agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/1] Remove support for COPY FROM with protocol version 2.
43+ messages / 7 participants
[nested] [flat]
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* [PATCH 1/1] Remove support for COPY FROM with protocol version 2.
@ 2021-02-03 15:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Heikki Linnakangas @ 2021-02-03 15:40 UTC (permalink / raw)
I'm working on a patch to refactor the way the encoding conversion is
performed, so that we convert the data in larger chunks, before scanning
the input for line boundaries. We can't do that, if we cannot safely try
to read ahead data past the end-of-copy marker. With the old protocol
gone, we can safely read as much as we want.
---
src/backend/commands/copyfrom.c | 7 -
src/backend/commands/copyfromparse.c | 162 +++++++----------------
src/backend/commands/copyto.c | 2 +-
src/include/commands/copyfrom_internal.h | 5 +-
4 files changed, 50 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index c39cc736ed2..6d43d056cca 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1125,13 +1125,6 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
- /*
- * In the old protocol, tell pqcomm that we can process normal protocol
- * messages again.
- */
- if (cstate->copy_src == COPY_OLD_FE)
- pq_endmsgread();
-
/* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 4c74067f849..e8497cbdf00 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -46,21 +46,6 @@
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
*/
-/*
- * This keeps the character read at the top of the loop in the buffer
- * even if there is more than one read-ahead.
- */
-#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
-if (1) \
-{ \
- if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
- { \
- raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
- need_data = true; \
- continue; \
- } \
-} else ((void) 0)
-
/* This consumes the remainder of the buffer and breaks */
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
if (1) \
@@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static bool CopyLoadRawBuf(CopyFromState cstate);
+static bool CopyLoadRawBuf(CopyFromState cstate, int minread);
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
@@ -144,14 +129,9 @@ ReceiveCopyBegin(CopyFromState cstate)
else
{
/* old way */
- if (cstate->opts.binary)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
- pq_putemptymessage('G');
- /* any error in old protocol will make us lose sync */
- pq_startmsgread();
- cstate->copy_src = COPY_OLD_FE;
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY FROM STDIN is not supported in protocol version 2")));
}
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -225,27 +205,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
- if (bytesread == 0)
+ if (bytesread < maxread)
cstate->reached_eof = true;
break;
- case COPY_OLD_FE:
-
- /*
- * We cannot read more than minread bytes (which in practice is 1)
- * because old protocol doesn't have any clear way of separating
- * the COPY stream from following data. This is slow, but not any
- * slower than the code path was originally, and we don't care
- * much anymore about the performance of old protocol.
- */
- if (pq_getbytes((char *) databuf, minread))
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection with an open transaction")));
- }
- bytesread = minread;
- break;
case COPY_NEW_FE:
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
@@ -312,6 +274,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case COPY_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
+ if (bytesread < minread)
+ cstate->reached_eof = true;
break;
}
@@ -363,14 +327,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
/*
* CopyLoadRawBuf loads some more data into raw_buf
*
- * Returns true if able to obtain at least one more byte, else false.
+ * Returns true if able to obtain at least 'minread' bytes, else false.
*
* If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
- * of the buffer and then we load more data after that. This case occurs only
- * when a multibyte character crosses a bufferload boundary.
+ * of the buffer and then we load more data after that.
*/
static bool
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromState cstate, int minread)
{
int nbytes = RAW_BUF_BYTES(cstate);
int inbytes;
@@ -381,14 +344,15 @@ CopyLoadRawBuf(CopyFromState cstate)
nbytes);
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
- 1, RAW_BUF_SIZE - nbytes);
+ minread, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
cstate->bytes_processed += nbytes;
pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
- return (inbytes > 0);
+
+ return (inbytes >= minread);
}
/*
@@ -423,7 +387,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
/* Load more data if buffer is empty. */
if (RAW_BUF_BYTES(cstate) == 0)
{
- if (!CopyLoadRawBuf(cstate))
+ if (!CopyLoadRawBuf(cstate, 1))
break; /* EOF */
}
@@ -619,21 +583,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
if (fld_count == -1)
{
/*
- * Received EOF marker. In a V3-protocol copy, wait for the
- * protocol-level EOF, and complain if it doesn't come
- * immediately. This ensures that we correctly handle CopyFail,
- * if client chooses to send that now.
+ * Received EOF marker. Wait for the protocol-level EOF, and
+ * complain if it doesn't come immediately. This ensures that we
+ * correctly handle CopyFail, if client chooses to send that now.
*
- * Note that we MUST NOT try to read more data in an old-protocol
- * copy, since there is no protocol-level EOF marker then. We
- * could go either way for copy from file, but choose to throw
- * error if there's data after the EOF marker, for consistency
- * with the new-protocol case.
+ * When copying from file, we could continue reading like we do in
+ * text mode, but we choose to throw error if there's data after
+ * the EOF marker, for consistency with the V3-protocol case.
*/
char dummy;
- if (cstate->copy_src != COPY_OLD_FE &&
- CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -717,7 +677,7 @@ CopyReadLine(CopyFromState cstate)
do
{
cstate->raw_buf_index = cstate->raw_buf_len;
- } while (CopyLoadRawBuf(cstate));
+ } while (CopyLoadRawBuf(cstate, 1));
}
}
else
@@ -786,7 +746,6 @@ CopyReadLineText(CopyFromState cstate)
char *copy_raw_buf;
int raw_buf_ptr;
int copy_buf_len;
- bool need_data = false;
bool hit_eof = false;
bool result = false;
char mblen_str[2];
@@ -840,38 +799,41 @@ CopyReadLineText(CopyFromState cstate)
char c;
/*
- * Load more data if needed. Ideally we would just force four bytes
- * of read-ahead and avoid the many calls to
- * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
- * does not allow us to read too far ahead or we might read into the
- * next data, so we read-ahead only as far we know we can. One
- * optimization would be to read-ahead four byte here if
- * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it,
- * considering the size of the buffer.
+ * Load more data if needed.
+ *
+ * We look ahead max three bytes in the code below (for the sequence
+ * \.<CR><NL>). Make sure we have at least four bytes in the buffer,
+ * so that the rest of the code in the loop can just assume that the
+ * data is in the buffer. Note that we always guarantee that there is
+ * one \0 in the buffer, after last valid byte; the lookahead code
+ * below relies on that.
*/
- if (raw_buf_ptr >= copy_buf_len || need_data)
+#define COPY_READ_LINE_LOOKAHEAD 4
+ if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len)
{
- REFILL_LINEBUF;
+ if (!hit_eof)
+ {
+ REFILL_LINEBUF;
- /*
- * Try to read some more data. This will certainly reset
- * raw_buf_index to zero, and raw_buf_ptr must go with it.
- */
- if (!CopyLoadRawBuf(cstate))
- hit_eof = true;
- raw_buf_ptr = 0;
- copy_buf_len = cstate->raw_buf_len;
+ /*
+ * Try to read some more data. This will certainly reset
+ * raw_buf_index to zero, and raw_buf_ptr must go with it.
+ */
+ if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD))
+ hit_eof = true;
+ raw_buf_ptr = 0;
+ copy_buf_len = cstate->raw_buf_len;
+ }
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (copy_buf_len <= 0)
+ if (copy_buf_len - raw_buf_ptr <= 0)
{
result = true;
break;
}
- need_data = false;
}
/* OK to fetch a character */
@@ -880,20 +842,6 @@ CopyReadLineText(CopyFromState cstate)
if (cstate->opts.csv_mode)
{
- /*
- * If character is '\\' or '\r', we may need to look ahead below.
- * Force fetch of the next character if we don't already have it.
- * We need to do this before changing CSV state, in case one of
- * these characters is also the quote or escape character.
- *
- * Note: old-protocol does not like forced prefetch, but it's OK
- * here since we cannot validly be at EOF.
- */
- if (c == '\\' || c == '\r')
- {
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
- }
-
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
@@ -927,14 +875,9 @@ CopyReadLineText(CopyFromState cstate)
cstate->eol_type == EOL_CRNL)
{
/*
- * If need more data, go back to loop top to load it.
- *
- * Note that if we are at EOF, c will wind up as '\0' because
- * of the guaranteed pad of raw_buf.
+ * Look at the next character. If we're at EOF, c2 will wind up as
+ * '\0' because of the guaranteed pad of raw_buf.
*/
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
-
- /* get next char */
c = copy_raw_buf[raw_buf_ptr];
if (c == '\n')
@@ -1000,7 +943,6 @@ CopyReadLineText(CopyFromState cstate)
{
char c2;
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
IF_NEED_REFILL_AND_EOF_BREAK(0);
/* -----
@@ -1015,15 +957,8 @@ CopyReadLineText(CopyFromState cstate)
{
raw_buf_ptr++; /* consume the '.' */
- /*
- * Note: if we loop back for more data here, it does not
- * matter that the CSV state change checks are re-executed; we
- * will come back here with no important state changed.
- */
if (cstate->eol_type == EOL_CRNL)
{
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1047,8 +982,6 @@ CopyReadLineText(CopyFromState cstate)
}
}
- /* Get the next character */
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
/* if hit_eof, c2 will become '\0' */
c2 = copy_raw_buf[raw_buf_ptr++];
@@ -1126,7 +1059,6 @@ not_end_of_copy:
mblen_str[0] = c;
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
raw_buf_ptr += mblen - 1;
}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e04ec1e331b..edbd5d83a0f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -162,7 +162,7 @@ SendCopyBegin(CopyToState cstate)
if (cstate->opts.binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("COPY BINARY is not supported to stdout or from stdin")));
+ errmsg("COPY BINARY is not supported to stdout or from stdin in protocol version 2")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index e37942df391..afa70326137 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -24,7 +24,7 @@
typedef enum CopySource
{
COPY_FILE, /* from file (or a piped program) */
- COPY_OLD_FE, /* from frontend (2.0 protocol) */
+ /* protocol version 2 not supported with COPY FROM */
COPY_NEW_FE, /* from frontend (3.0 protocol) */
COPY_CALLBACK /* from callback function */
} CopySource;
@@ -71,8 +71,7 @@ typedef struct CopyFromStateData
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */
- bool reached_eof; /* true if we read to end of copy data (not
- * all copy_src types maintain this) */
+ bool reached_eof; /* true if we read to end of copy data */
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
--
2.30.0
--------------95F419C4E784A7684A2358D2--
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-01 06:43 Peter Smith <[email protected]>
0 siblings, 2 replies; 43+ messages in thread
From: Peter Smith @ 2022-07-01 06:43 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
Below are some review comments for patches v14-0001, and v14-0002:
========
v14-0001
========
1.1 Commit message
For now, 'parallel' means the streaming will be applied
via a apply background worker if available. 'on' means the streaming
transaction will be spilled to disk. By the way, we do not change the default
behaviour.
SUGGESTION (minor tweaks)
The parameter value 'parallel' means the streaming will be applied via
an apply background worker, if available. The parameter value 'on'
means the streaming transaction will be spilled to disk. The default
value is 'off' (same as current behaviour).
======
1.2 doc/src/sgml/protocol.sgml - Protocol constants
Previously I wrote that since there are protocol changes here,
shouldn’t there also be some corresponding LOGICALREP_PROTO_XXX
constants and special checking added in the worker.c?
But you said [1 comment #6] you think it is OK because...
IMO, I still disagree with the reply. The fact is that the protocol
*has* been changed, so IIUC that is precisely the reason for having
those protocol constants.
e.g I am guessing you might assign the new one somewhere here:
--
server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
options.proto.logical.proto_version =
server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
LOGICALREP_PROTO_VERSION_NUM;
--
And then later you would refer to this new protocol version (instead
of the server version) when calling to the apply_handle_stream_abort
function.
======
1.3 doc/src/sgml/ref/create_subscription.sgml
+ <para>
+ If set to <literal>on</literal>, the changes of transaction are
+ written to temporary files and then applied at once after the
+ transaction is committed on the publisher.
+ </para>
Previously I suggested changing some text but it was rejected [1
comment #8] because you said there may be *multiple* files, not just
one. That is fair enough, but there were some other changes to that
suggested text unrelated to the number of files.
SUGGESTION #2
If set to on, the incoming changes are written to temporary files and
then applied only after the transaction is committed on the publisher.
~~~
1.4
+ <para>
+ If set to <literal>parallel</literal>, incoming changes are directly
+ applied via one of the apply background workers, if available. If no
+ background worker is free to handle streaming transaction then the
+ changes are written to a file and applied after the transaction is
+ committed. Note that if an error happens when applying changes in a
+ background worker, the finish LSN of the remote transaction might
+ not be reported in the server log.
</para>
Should this also say "written to temporary files" instead of "written
to a file"?
======
1.5 src/backend/commands/subscriptioncmds.c
+ /*
+ * If no parameter given, assume "true" is meant.
+ */
Previously I suggested an update for this comment, but it was rejected
[1 comment #12] saying you wanted consistency with defGetBoolean.
Sure, that is one point of view. Another one is that "two wrongs don't
make a right". IIUC that comment as it currently stands is incorrect
because in this case there *is* a parameter given - it is just the
parameter *value* that is missing. Maybe see what other people think?
======
1.6. src/backend/replication/logical/Makefile
It seems to me like these files were intended to be listed in
alphabetical order, so you should move this new file accordingly.
======
1.7 .../replication/logical/applybgworker.c
+/* queue size of DSM, 16 MB for now. */
+#define DSM_QUEUE_SIZE 160000000
The comment should start uppercase.
~~~
1.8 .../replication/logical/applybgworker.c - apply_bgworker_can_start
Maybe this is just my opinion but it sounds a bit strange to over-use
"we" in all the comments.
1.8.a
+/*
+ * Confirm if we can try to start a new apply background worker.
+ */
+static bool
+apply_bgworker_can_start(TransactionId xid)
SUGGESTION
Check if starting a new apply background worker is allowed.
1.8.b
+ /*
+ * We don't start new background worker if we are not in streaming parallel
+ * mode.
+ */
SUGGESTION
Don't start a new background worker if not in streaming parallel mode.
1.8.c
+ /*
+ * We don't start new background worker if user has set skiplsn as it's
+ * possible that user want to skip the streaming transaction. For
+ * streaming transaction, we need to spill the transaction to disk so that
+ * we can get the last LSN of the transaction to judge whether to skip
+ * before starting to apply the change.
+ */
SUGGESTION
Don't start a new background worker if...
~~~
1.9 .../replication/logical/applybgworker.c - apply_bgworker_start
+/*
+ * Try to start worker inside ApplyWorkersHash for requested xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_start(TransactionId xid)
The comment seems not quite right.
SUGGESTION
Try to start an apply background worker and, if successful, cache it
in ApplyWorkersHash keyed by the specified xid.
~~~
1.10 .../replication/logical/applybgworker.c - apply_bgworker_find
+ /*
+ * Find entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
+ if (found)
+ {
+ entry->wstate->pstate->status = APPLY_BGWORKER_BUSY;
+ return entry->wstate;
+ }
+ else
+ return NULL;
+}
IMO it is an unexpected side-effect for the function called "find" to
be also modifying the thing that it found. IMO this setting BUSY
should either be done by the caller, or else this function name should
be renamed to make it obvious that this is doing more than just
"finding" something.
~~~
1.11 .../replication/logical/applybgworker.c - LogicalApplyBgwLoop
+ /*
+ * Push apply error context callback. Fields will be filled applying
+ * applying a change.
+ */
Typo: "applying applying"
~~~
1.12 .../replication/logical/applybgworker.c - apply_bgworker_setup
+ if (launched)
+ ApplyWorkersList = lappend(ApplyWorkersList, wstate);
+ else
+ {
+ shm_mq_detach(wstate->mq_handle);
+ dsm_detach(wstate->dsm_seg);
+ pfree(wstate);
+
+ wstate->mq_handle = NULL;
+ wstate->dsm_seg = NULL;
+ wstate = NULL;
+ }
I am not sure what those first 2 NULL assignments are trying to
achieve. Nothing AFAICT. In any case, it looks like a bug to deference
the 'wstate' after you already pfree-d it in the line above.
~~~
1.13 .../replication/logical/applybgworker.c - apply_bgworker_check_status
+ * Exit if any relation is not in the READY state and if any worker is handling
+ * the streaming transaction at the same time. Because for streaming
+ * transactions that is being applied in apply background worker, we cannot
+ * decide whether to apply the change for a relation that is not in the READY
+ * state (see should_apply_changes_for_rel) as we won't know remote_final_lsn
+ * by that time.
+ */
+void
+apply_bgworker_check_status(void)
Somehow, I felt that this "Exit if..." comment really belonged at the
appropriate place in the function body, instead of in the function
header.
======
1.14 src/backend/replication/logical/launcher.c - WaitForReplicationWorkerAttach
@@ -151,8 +153,10 @@ get_subscription_list(void)
*
* This is only needed for cleaning up the shared memory in case the worker
* fails to attach.
+ *
+ * Returns false if the attach fails. Otherwise return true.
*/
-static void
+static bool
WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
Comment should say either "Return" or "returns"; not one of each.
~~~
1.15. src/backend/replication/logical/launcher.c -
WaitForReplicationWorkerAttach
+ return worker->in_use ? true : false;
Same as just:
return worker->in_use;
~~~
1.16. src/backend/replication/logical/launcher.c - logicalrep_worker_launch
+ bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
+
+ /* We don't support table sync in subworker */
+ Assert(!(is_subworker && OidIsValid(relid)));
I'm not sure the comment is good. It sounds like it is something that
might be possible but is just current "not supported". In fact, I
thought this is really just a sanity check because the combination of
those params is just plain wrong isn't it? Maybe a better comment is
just:
/* Sanity check */
======
1.17 src/backend/replication/logical/proto.c
+ /*
+ * If the version of the publisher is lower than the version of the
+ * subscriber, it may not support sending these two fields. So these
+ * fields are only taken if they are included.
+ */
+ if (include_abort_lsn)
1.17a
I thought that the comment about "versions of publishers lower than
version of subscribers..." is bogus. Perhaps you have in mind just
thinking about versions prior to PG16 but that is not what the comment
is saying. E.g. sometime in the future, the publisher may be PG18 and
the subscriber might be PG25. So that might work fine (even though the
publisher is a lower version), but this comment will be completely
misleading. BTW this is another reason I think code needs to be using
protocol versions (not server versions). [See other comment #1.2]
1.17b.
Anyway, I felt that any comment describing the meaning of the the
'include_abort_lsn' param would be better in the function header
comment, instead of in the function body.
======
1.18 src/backend/replication/logical/worker.c - file header comment
+ * 1) Separate background workers
+ *
+ * Assign a new apply background worker (if available) as soon as the xact's...
Somehow this long comment did not ever mention that this mode is
selected by the user using the 'streaming=parallel'. I thought
probably it should say that somewhere here.
~~~
1.19. src/backend/replication/logical/worker.c -
ApplyErrorCallbackArg apply_error_callback_arg =
{
.command = 0,
.rel = NULL,
.remote_attnum = -1,
.remote_xid = InvalidTransactionId,
.finish_lsn = InvalidXLogRecPtr,
.origin_name = NULL,
};
I still thought that the above initialization deserves some sort of
comment, even if you don't want to use the comment text previously
suggested [1 comment #41]
~~~
1.20 src/backend/replication/logical/worker.c -
@@ -251,27 +258,38 @@ static MemoryContext LogicalStreamingContext = NULL;
WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
Subscription *MySubscription = NULL;
-static bool MySubscriptionValid = false;
+bool MySubscriptionValid = false;
bool in_remote_transaction = false;
static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
/* fields valid only when processing streamed transaction */
-static bool in_streamed_transaction = false;
+bool in_streamed_transaction = false;
The tab alignment here looks wrong. IMO it's not worth trying to align
these at all. I think the tabs are leftover from before when the vars
used to be static.
~~~
1.21 src/backend/replication/logical/worker.c - apply_bgworker_active
+/* Check if we are applying the transaction in apply background worker */
+#define apply_bgworker_active() (in_streamed_transaction &&
stream_apply_worker != NULL)
Sorry [1 comment #42b], I had meant to write "in apply background
worker" -> "in an apply background worker".
~~~
1.22 src/backend/replication/logical/worker.c - skip_xact_finish_lsn
/*
* We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
* the subscription if the remote transaction's finish LSN matches
the subskiplsn.
* Once we start skipping changes, we don't stop it until we skip all
changes of
* the transaction even if pg_subscription is updated and
MySubscription->skiplsn
- * gets changed or reset during that. Also, in streaming transaction cases, we
- * don't skip receiving and spooling the changes since we decide whether or not
+ * gets changed or reset during that. Also, in streaming transaction
cases (streaming = on),
+ * we don't skip receiving and spooling the changes since we decide
whether or not
* to skip applying the changes when starting to apply changes. The
subskiplsn is
* cleared after successfully skipping the transaction or applying non-empty
* transaction. The latter prevents the mistakenly specified subskiplsn from
- * being left.
+ * being left. Note that we cannot skip the streaming transaction in parallel
+ * mode, because we cannot get the finish LSN before applying the changes.
*/
"in parallel mode, because" -> "in 'streaming = parallel' mode, because"
~~~
1.23 src/backend/replication/logical/worker.c - handle_streamed_transaction
1.23a
/*
- * Handle streamed transactions.
+ * Handle streamed transactions for both main apply worker and apply background
+ * worker.
SUGGESTION
Handle streamed transactions for both the main apply worker and the
apply background workers.
1.23b
+ * In streaming case (receiving a block of streamed transaction), for
+ * SUBSTREAM_ON mode, we simply redirect it to a file for the proper toplevel
+ * transaction, and for SUBSTREAM_PARALLEL mode, we send the changes to
+ * background apply worker (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE
+ * changes will also be applied in main apply worker).
"background apply worker" -> "apply background workers"
Also, I think you don't need to say "we" everywhere:
"we simply redirect it" -> "simply redirect it"
"we send the changes" -> "send the changes"
1.23c.
+ * But there are two exceptions: If we apply streamed transaction in main apply
+ * worker with parallel mode, it will return false when we address
+ * LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes.
SUGGESTION
Exception: When parallel mode is applying streamed transaction in the
main apply worker, (e.g. when addressing
LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes), then return false.
~~~
1.24 src/backend/replication/logical/worker.c - handle_streamed_transaction
1.24a.
/* not in streaming mode */
- if (!in_streamed_transaction)
+ if (!(in_streamed_transaction || am_apply_bgworker()))
return false;
Uppercase comment
1.24b
+ /* define a savepoint for a subxact if needed. */
+ apply_bgworker_subxact_info_add(current_xid);
Uppercase comment
~~~
1.25 src/backend/replication/logical/worker.c - handle_streamed_transaction
+ /*
+ * This is the main apply worker, and there is an apply background
+ * worker. So we apply the changes of this transaction in an apply
+ * background worker. Pass the data to the worker.
+ */
SUGGESTION (to be more consistent with the next comment)
This is the main apply worker, but there is an apply background
worker, so apply the changes of this transaction in that background
worker. Pass the data to the worker.
~~~
1.26 src/backend/replication/logical/worker.c - handle_streamed_transaction
+ /*
+ * This is the main apply worker, but there is no apply background
+ * worker. So we write to temporary files and apply when the final
+ * commit arrives.
SUGGESTION
This is the main apply worker, but there is no apply background
worker, so write to temporary files and apply when the final commit
arrives.
~~~
1.27 src/backend/replication/logical/worker.c - apply_handle_stream_prepare
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker.
+ */
SUGGESTION:
Check if we are processing this transaction in an apply background
worker and if so, send the changes to that worker.
~~~
1.28 src/backend/replication/logical/worker.c - apply_handle_stream_prepare
+ if (wstate)
+ {
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * Wait for apply background worker to finish. This is required to
+ * maintain commit order which avoids failures due to transaction
+ * dependencies and deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
I think maybe the comment can be changed slightly, and then it can
move up one line to the top of this code block (above the 3
statements). I think it will become more readable.
SUGGESTION
After sending the data to the apply background worker, wait for that
worker to finish. This is necessary to maintain commit order which
avoids failures due to transaction dependencies and deadlocks.
~~~
1.29 src/backend/replication/logical/worker.c - apply_handle_stream_start
+ /*
+ * If no worker is available for the first stream start, we start to
+ * serialize all the changes of the transaction.
+ */
+ else
+ {
1.29a.
I felt that this comment should be INSIDE the else { block to be more readable.
1.29b.
The comment can also be simplified a bit
SUGGESTION:
Since no apply background worker is available for the first stream
start, serialize all the changes of the transaction.
~~~
1.30 src/backend/replication/logical/worker.c - apply_handle_stream_start
+ /* if this is not the first segment, open existing subxact file */
+ if (!first_segment)
+ subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
Uppercase comment
~~~
1.31. src/backend/replication/logical/worker.c - apply_handle_stream_stop
+ if (apply_bgworker_active())
+ {
+ char action = LOGICAL_REP_MSG_STREAM_STOP;
Are all the tabs before the variable needed?
~~~
1.32. src/backend/replication/logical/worker.c - apply_handle_stream_abort
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ include_abort_lsn = MyParallelState->server_version >= 150000;
Previously I already reported about this [1 comment #50]
I just do not trust this code to do the correct thing. E.g. what if
streaming=parallel but all bgworkers are exhausted then IIUC the
am_apply_bgworker() will not be true. But then with both PG15 servers
for pub/sub you will WRITE something but then you will not READ it.
Won't the stream IO will get out of step and everything will fall
apart?
Perhaps the include_abort_lsn assignment should be unconditionally
set, and I think this should be a protocol version check instead of a
server version check shouldn’t it (see my earlier comment 1.2)
~~~
1.32 src/backend/replication/logical/worker.c - apply_handle_stream_abort
BTW, I think the PG16devel is now stamped in the GitHub HEAD so
perhaps all of your 150000 checks should be now changed to say 160000?
~~~
1.33 src/backend/replication/logical/worker.c - apply_handle_stream_abort
+ /*
+ * We are in main apply worker and the transaction has been serialized
+ * to file.
+ */
+ else
+ serialize_stream_abort(xid, subxid);
I think this will be more readable if written like:
else
{
/* put comment here... */
serialize_stream_abort(xid, subxid);
}
~~~
1.34 src/backend/replication/logical/worker.c - apply_dispatch
-
/*
* Logical replication protocol message dispatcher.
*/
-static void
+void
apply_dispatch(StringInfo s)
Maybe removing the whitespace is not really needed as part of this patch?
======
1.35 src/include/catalog/pg_subscription.h
+/* Disallow streaming in-progress transactions */
+#define SUBSTREAM_OFF 'f'
+
+/*
+ * Streaming transactions are written to a temporary file and applied only
+ * after the transaction is committed on upstream.
+ */
+#define SUBSTREAM_ON 't'
+
+/* Streaming transactions are applied immediately via a background worker */
+#define SUBSTREAM_PARALLEL 'p'
+
1.35a
Should all these "Streaming transactions" be called "Streaming
in-progress transactions"?
1.35b.
Either align the values or don’t. Currently, they seem half-aligned.
1.35c.
SUGGESTION (modify the 1st comment to be more consistent with the others)
Streaming in-progress transactions are disallowed.
======
1.36 src/include/replication/worker_internal.h
extern int logicalrep_sync_worker_count(Oid subid);
+extern int logicalrep_apply_background_worker_count(Oid subid);
Just wondering if this should be called
"logicalrep_apply_bgworker_count(Oid subid);" for consistency with the
other function naming.
========
v14-0002
========
2.1 Commit message
Change all TAP tests using the SUBSCRIPTION "streaming" option, so they
now test both 'on' and 'parallel' values.
"option" -> "parameter"
------
[1] https://www.postgresql.org/message-id/OS3PR01MB6275DCCDF35B3BBD52CA02CC9EB89%40OS3PR01MB6275.jpnprd0...
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-01 09:43 Amit Kapila <[email protected]>
parent: Peter Smith <[email protected]>
1 sibling, 1 reply; 43+ messages in thread
From: Amit Kapila @ 2022-07-01 09:43 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 1, 2022 at 12:13 PM Peter Smith <[email protected]> wrote:
>
> ======
>
> 1.2 doc/src/sgml/protocol.sgml - Protocol constants
>
> Previously I wrote that since there are protocol changes here,
> shouldn’t there also be some corresponding LOGICALREP_PROTO_XXX
> constants and special checking added in the worker.c?
>
> But you said [1 comment #6] you think it is OK because...
>
> IMO, I still disagree with the reply. The fact is that the protocol
> *has* been changed, so IIUC that is precisely the reason for having
> those protocol constants.
>
> e.g I am guessing you might assign the new one somewhere here:
> --
> server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
> options.proto.logical.proto_version =
> server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
> server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
> LOGICALREP_PROTO_VERSION_NUM;
> --
>
> And then later you would refer to this new protocol version (instead
> of the server version) when calling to the apply_handle_stream_abort
> function.
>
> ======
>
One point related to this that occurred to me is how it will behave if
the publisher is of version >=16 whereas the subscriber is of versions
<=15? Won't in that case publisher sends the new fields but
subscribers won't be reading those which may cause some problems.
> ======
>
> 1.5 src/backend/commands/subscriptioncmds.c
>
> + /*
> + * If no parameter given, assume "true" is meant.
> + */
>
> Previously I suggested an update for this comment, but it was rejected
> [1 comment #12] saying you wanted consistency with defGetBoolean.
>
> Sure, that is one point of view. Another one is that "two wrongs don't
> make a right". IIUC that comment as it currently stands is incorrect
> because in this case there *is* a parameter given - it is just the
> parameter *value* that is missing.
>
You have a point but if we see this function in the vicinity then the
proposed comment also makes sense.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-07 03:44 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
1 sibling, 1 reply; 43+ messages in thread
From: [email protected] @ 2022-07-07 03:44 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 1, 2022 at 14:43 PM Peter Smith <[email protected]> wrote:
> Below are some review comments for patches v14-0001, and v14-0002:
Thanks for your comments.
> 1.10 .../replication/logical/applybgworker.c - apply_bgworker_find
>
> + /*
> + * Find entry for requested transaction.
> + */
> + entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
> + if (found)
> + {
> + entry->wstate->pstate->status = APPLY_BGWORKER_BUSY;
> + return entry->wstate;
> + }
> + else
> + return NULL;
> +}
>
> IMO it is an unexpected side-effect for the function called "find" to
> be also modifying the thing that it found. IMO this setting BUSY
> should either be done by the caller, or else this function name should
> be renamed to make it obvious that this is doing more than just
> "finding" something.
Since we set the state to BUSY in the function apply_bgworker_start and the
state is not modified (set to FINISHED) until the transaction completes, I
think we do not need to set this state to BUSY again in the function
apply_bgworker_find during applying the transaction.
So I removed it and invoked function Assert.
I also invoked function Assert in function apply_bgworker_start.
> 1.16. src/backend/replication/logical/launcher.c - logicalrep_worker_launch
>
> + bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
> +
> + /* We don't support table sync in subworker */
> + Assert(!(is_subworker && OidIsValid(relid)));
>
> I'm not sure the comment is good. It sounds like it is something that
> might be possible but is just current "not supported". In fact, I
> thought this is really just a sanity check because the combination of
> those params is just plain wrong isn't it? Maybe a better comment is
> just:
> /* Sanity check */
Improved this comment as following:
```
/* Sanity check : we don't support table sync in subworker. */
```
> 1.22 src/backend/replication/logical/worker.c - skip_xact_finish_lsn
>
> /*
> * We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
> * the subscription if the remote transaction's finish LSN matches
> the subskiplsn.
> * Once we start skipping changes, we don't stop it until we skip all
> changes of
> * the transaction even if pg_subscription is updated and
> MySubscription->skiplsn
> - * gets changed or reset during that. Also, in streaming transaction cases, we
> - * don't skip receiving and spooling the changes since we decide whether or not
> + * gets changed or reset during that. Also, in streaming transaction
> cases (streaming = on),
> + * we don't skip receiving and spooling the changes since we decide
> whether or not
> * to skip applying the changes when starting to apply changes. The
> subskiplsn is
> * cleared after successfully skipping the transaction or applying non-empty
> * transaction. The latter prevents the mistakenly specified subskiplsn from
> - * being left.
> + * being left. Note that we cannot skip the streaming transaction in parallel
> + * mode, because we cannot get the finish LSN before applying the changes.
> */
>
> "in parallel mode, because" -> "in 'streaming = parallel' mode, because"
Not sure about this.
> 1.28 src/backend/replication/logical/worker.c - apply_handle_stream_prepare
>
> + if (wstate)
> + {
> + apply_bgworker_send_data(wstate, s->len, s->data);
> +
> + /*
> + * Wait for apply background worker to finish. This is required to
> + * maintain commit order which avoids failures due to transaction
> + * dependencies and deadlocks.
> + */
> + apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
> + apply_bgworker_free(wstate);
>
> I think maybe the comment can be changed slightly, and then it can
> move up one line to the top of this code block (above the 3
> statements). I think it will become more readable.
>
> SUGGESTION
> After sending the data to the apply background worker, wait for that
> worker to finish. This is necessary to maintain commit order which
> avoids failures due to transaction dependencies and deadlocks.
I think it might be better to add a new comment before invoking function
apply_bgworker_send_data. Improve the comments as you suggested.
I improved this point in function apply_handle_stream_prepare,
apply_handle_stream_abort and apply_handle_stream_commit. What do you think
about changing it like this:
```
/* Send STREAM PREPARE message to the apply background worker. */
apply_bgworker_send_data(wstate, s->len, s->data);
/*
* After sending the data to the apply background worker, wait for
* that worker to finish. This is necessary to maintain commit
* order which avoids failures due to transaction dependencies and
* deadlocks.
*/
apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
```
> 1.34 src/backend/replication/logical/worker.c - apply_dispatch
>
> -
> /*
> * Logical replication protocol message dispatcher.
> */
> -static void
> +void
> apply_dispatch(StringInfo s)
>
> Maybe removing the whitespace is not really needed as part of this patch?
Yes, this change is not necessary for this patch.
But since this change does not involve the modification of comments and actual
code, it just adjusts the blank line between the function modified by this
patch and the previous function, so I think it is okay in this patch.
> 2.1 Commit message
>
> Change all TAP tests using the SUBSCRIPTION "streaming" option, so they
> now test both 'on' and 'parallel' values.
>
> "option" -> "parameter"
Sorry I missed this point when I was merging the patches. I merged this change
in v15.
Attach the new patches.
Also improved the patches as suggested in [1], [2] and [3].
[1] - https://www.postgresql.org/message-id/CAA4eK1KgovaRcbSuzzWki1HVso6oLAdZ2aPr1nWxX1x%3DVDBQJg%40mail.g...
[2] - https://www.postgresql.org/message-id/CAHut%2BPtRNAOwFtBp_TnDWdC7UpcTxPJzQnrm%3DNytN7cVBt5zRQ%40mail...
[3] - https://www.postgresql.org/message-id/CAHut%2BPvrw%2BtgCEYGxv%2BnKrqg-zbJdYEXee6o4irPAsYoXcuUcw%40ma...
Regards,
Wang wei
Attachments:
[application/octet-stream] v15-0001-Perform-streaming-logical-transactions-by-backgr.patch (105.6K, ../../OS3PR01MB62755C6C9A75EB09F7218B589E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-v15-0001-Perform-streaming-logical-transactions-by-backgr.patch)
download | inline diff:
From 10ac62669db230d73f5e0f65fb52d01649b1ed6c Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Wed, 20 Apr 2022 16:45:07 +0800
Subject: [PATCH v15 1/4] Perform streaming logical transactions by background
workers
Currently, for large transactions, the publisher sends the data in multiple
streams (changes divided into chunks depending upon logical_decoding_work_mem),
and then on the subscriber-side, the apply worker writes the changes into
temporary files and once it receives the commit, it reads from the file and
applies the entire transaction. To improve the performance of such
transactions, we can instead allow them to be applied via background workers.
In this approach, we assign a new apply background worker (if available) as
soon as the xact's first stream is received and the main apply worker will send
changes to this new worker via shared memory. The apply background worker will
directly apply the change instead of writing it to temporary files. We keep
this worker assigned till the transaction commit is received and also wait for
the worker to finish at commit. This preserves commit ordering and avoids
writing to and reading from file in most cases. We still need to spill if there
is no worker available.
This patch also extends the SUBSCRIPTION 'streaming' parameter so that the user
can control whether to apply the streaming transaction in an apply background
worker or spill the change to disk. The user can set the streaming parameter to
'on/off', 'parallel'. The parameter value 'parallel' means the streaming will
be applied via an apply background worker, if available. The parameter value
'on' means the streaming transaction will be spilled to disk. The default value
is 'off' (same as current behaviour).
---
doc/src/sgml/catalogs.sgml | 10 +-
doc/src/sgml/config.sgml | 25 +
doc/src/sgml/logical-replication.sgml | 10 +
doc/src/sgml/protocol.sgml | 19 +
doc/src/sgml/ref/create_subscription.sgml | 24 +-
src/backend/access/transam/xact.c | 13 +
src/backend/commands/subscriptioncmds.c | 66 +-
src/backend/postmaster/bgworker.c | 3 +
src/backend/replication/logical/Makefile | 1 +
.../replication/logical/applybgworker.c | 775 ++++++++++++++++++
src/backend/replication/logical/decode.c | 10 +-
src/backend/replication/logical/launcher.c | 130 ++-
src/backend/replication/logical/origin.c | 26 +-
src/backend/replication/logical/proto.c | 41 +-
.../replication/logical/reorderbuffer.c | 10 +-
src/backend/replication/logical/tablesync.c | 10 +-
src/backend/replication/logical/worker.c | 688 ++++++++++++----
src/backend/replication/pgoutput/pgoutput.c | 9 +-
src/backend/utils/activity/wait_event.c | 3 +
src/backend/utils/misc/guc.c | 12 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/bin/pg_dump/pg_dump.c | 6 +-
src/include/catalog/pg_subscription.h | 21 +-
src/include/replication/logicallauncher.h | 1 +
src/include/replication/logicalproto.h | 27 +-
src/include/replication/logicalworker.h | 1 +
src/include/replication/origin.h | 2 +-
src/include/replication/reorderbuffer.h | 7 +-
src/include/replication/worker_internal.h | 102 ++-
src/include/utils/wait_event.h | 1 +
src/test/regress/expected/subscription.out | 2 +-
src/tools/pgindent/typedefs.list | 5 +
32 files changed, 1841 insertions(+), 220 deletions(-)
create mode 100644 src/backend/replication/logical/applybgworker.c
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 4f3f375a84..815cae6082 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7873,11 +7873,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry role="catalog_table_entry"><para role="column_definition">
- <structfield>substream</structfield> <type>bool</type>
+ <structfield>substream</structfield> <type>char</type>
</para>
<para>
- If true, the subscription will allow streaming of in-progress
- transactions
+ Controls how to handle the streaming of in-progress transactions:
+ <literal>f</literal> = disallow streaming of in-progress transactions,
+ <literal>t</literal> = spill the changes of in-progress transactions to
+ disk and apply at once after the transaction is committed on the
+ publisher,
+ <literal>p</literal> = apply changes directly using a background worker
</para></entry>
</row>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..6bbd986195 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4970,6 +4970,31 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
</listitem>
</varlistentry>
+ <varlistentry id="guc-max-apply-bgworkers-per-subscription" xreflabel="max_apply_bgworkers_per_subscription">
+ <term><varname>max_apply_bgworkers_per_subscription</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>max_apply_bgworkers_per_subscription</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Maximum number of apply background workers per subscription. This
+ parameter controls the amount of parallelism of the streaming of
+ in-progress transactions when subscription parameter
+ <literal>streaming = parallel</literal>.
+ </para>
+ <para>
+ The apply background workers are taken from the pool defined by
+ <varname>max_logical_replication_workers</varname>.
+ </para>
+ <para>
+ The default value is 2. This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command
+ line.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</sect2>
diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml
index bdf1e7b727..92997f9299 100644
--- a/doc/src/sgml/logical-replication.sgml
+++ b/doc/src/sgml/logical-replication.sgml
@@ -1153,6 +1153,16 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER
might not violate any constraint. This can easily make the subscriber
inconsistent.
</para>
+
+ <para>
+ When the streaming mode is <literal>parallel</literal>, the finish LSN of
+ failed transactions may not be logged. In that case, it may be necessary to
+ change the streaming mode to <literal>on</literal> and cause the same
+ conflicts again so the finish LSN of the failed transaction will be written
+ to the server log. For the usage of finish LSN, please refer to <link
+ linkend="sql-altersubscription"><command>ALTER SUBSCRIPTION ...
+ SKIP</command></link>.
+ </para>
</sect1>
<sect1 id="logical-replication-restrictions">
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..7e88ba9631 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -6809,6 +6809,25 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term>Int64 (XLogRecPtr)</term>
+ <listitem>
+ <para>
+ The LSN of the abort.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>Int64 (TimestampTz)</term>
+ <listitem>
+ <para>
+ Abort timestamp of the transaction. The value is in number
+ of microseconds since PostgreSQL epoch (2000-01-01).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term>Int32 (TransactionId)</term>
<listitem>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 34b3264b26..71dd4aca81 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -217,13 +217,29 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</varlistentry>
<varlistentry>
- <term><literal>streaming</literal> (<type>boolean</type>)</term>
+ <term><literal>streaming</literal> (<type>enum</type>)</term>
<listitem>
<para>
Specifies whether to enable streaming of in-progress transactions
- for this subscription. By default, all transactions
- are fully decoded on the publisher and only then sent to the
- subscriber as a whole.
+ for this subscription. The default value is <literal>off</literal>,
+ meaning all transactions are fully decoded on the publisher and only
+ then sent to the subscriber as a whole.
+ </para>
+
+ <para>
+ If set to <literal>on</literal>, the incoming changes are written to
+ temporary files and then applied only after the transaction is
+ committed on the publisher.
+ </para>
+
+ <para>
+ If set to <literal>parallel</literal>, incoming changes are directly
+ applied via one of the apply background workers, if available. If no
+ background worker is free to handle streaming transaction then the
+ changes are written to temporary files and applied after the
+ transaction is committed. Note that if an error happens when
+ applying changes in a background worker, the finish LSN of the
+ remote transaction might not be reported in the server log.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 116de1175b..3e61a57b50 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -1711,6 +1711,7 @@ RecordTransactionAbort(bool isSubXact)
int nchildren;
TransactionId *children;
TimestampTz xact_time;
+ bool replorigin;
/*
* If we haven't been assigned an XID, nobody will care whether we aborted
@@ -1741,6 +1742,13 @@ RecordTransactionAbort(bool isSubXact)
elog(PANIC, "cannot abort transaction %u, it was already committed",
xid);
+ /*
+ * Are we using the replication origins feature? Or, in other words,
+ * are we replaying remote actions?
+ */
+ replorigin = (replorigin_session_origin != InvalidRepOriginId &&
+ replorigin_session_origin != DoNotReplicateId);
+
/* Fetch the data we need for the abort record */
nrels = smgrGetPendingDeletes(false, &rels);
nchildren = xactGetCommittedChildren(&children);
@@ -1765,6 +1773,11 @@ RecordTransactionAbort(bool isSubXact)
MyXactFlags, InvalidTransactionId,
NULL);
+ if (replorigin)
+ /* Move LSNs forward for this replication origin */
+ replorigin_session_advance(replorigin_session_origin_lsn,
+ XactLastRecEnd);
+
/*
* Report the latest async abort LSN, so that the WAL writer knows to
* flush this abort. There's nothing to be gained by delaying this, since
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bdc1208724..5f349067cc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -83,7 +83,7 @@ typedef struct SubOpts
bool copy_data;
bool refresh;
bool binary;
- bool streaming;
+ char streaming;
bool twophase;
bool disableonerr;
XLogRecPtr lsn;
@@ -95,6 +95,62 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub,
static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
+/*
+ * Extract the streaming mode value from a DefElem. This is like
+ * defGetBoolean() but also accepts the special value of "parallel".
+ */
+static char
+defGetStreamingMode(DefElem *def)
+{
+ /*
+ * If no value given, assume "true" is meant.
+ */
+ if (def->arg == NULL)
+ return SUBSTREAM_ON;
+
+ /*
+ * Allow 0, 1, "false", "true", "off", "on" or "parallel".
+ */
+ switch (nodeTag(def->arg))
+ {
+ case T_Integer:
+ switch (intVal(def->arg))
+ {
+ case 0:
+ return SUBSTREAM_OFF;
+ case 1:
+ return SUBSTREAM_ON;
+ default:
+ /* otherwise, error out below */
+ break;
+ }
+ break;
+ default:
+ {
+ char *sval = defGetString(def);
+
+ /*
+ * The set of strings accepted here should match up with the
+ * grammar's opt_boolean_or_string production.
+ */
+ if (pg_strcasecmp(sval, "false") == 0 ||
+ pg_strcasecmp(sval, "off") == 0)
+ return SUBSTREAM_OFF;
+ if (pg_strcasecmp(sval, "true") == 0 ||
+ pg_strcasecmp(sval, "on") == 0)
+ return SUBSTREAM_ON;
+ if (pg_strcasecmp(sval, "parallel") == 0)
+ return SUBSTREAM_PARALLEL;
+ }
+ break;
+ }
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("%s requires a Boolean value or \"parallel\"",
+ def->defname)));
+ return SUBSTREAM_OFF; /* keep compiler quiet */
+}
+
/*
* Common option parsing function for CREATE and ALTER SUBSCRIPTION commands.
*
@@ -132,7 +188,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
if (IsSet(supported_opts, SUBOPT_BINARY))
opts->binary = false;
if (IsSet(supported_opts, SUBOPT_STREAMING))
- opts->streaming = false;
+ opts->streaming = SUBSTREAM_OFF;
if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
opts->twophase = false;
if (IsSet(supported_opts, SUBOPT_DISABLE_ON_ERR))
@@ -233,7 +289,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
errorConflictingDefElem(defel, pstate);
opts->specified_opts |= SUBOPT_STREAMING;
- opts->streaming = defGetBoolean(defel);
+ opts->streaming = defGetStreamingMode(defel);
}
else if (strcmp(defel->defname, "two_phase") == 0)
{
@@ -600,7 +656,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner);
values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled);
values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary);
- values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming);
+ values[Anum_pg_subscription_substream - 1] = CharGetDatum(opts.streaming);
values[Anum_pg_subscription_subtwophasestate - 1] =
CharGetDatum(opts.twophase ?
LOGICALREP_TWOPHASE_STATE_PENDING :
@@ -1059,7 +1115,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
if (IsSet(opts.specified_opts, SUBOPT_STREAMING))
{
values[Anum_pg_subscription_substream - 1] =
- BoolGetDatum(opts.streaming);
+ CharGetDatum(opts.streaming);
replaces[Anum_pg_subscription_substream - 1] = true;
}
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 40601aefd9..40ccb8993c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -128,6 +128,9 @@ static const struct
},
{
"ApplyWorkerMain", ApplyWorkerMain
+ },
+ {
+ "ApplyBgworkerMain", ApplyBgworkerMain
}
};
diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile
index c4e2fdeb71..cbfb5d794e 100644
--- a/src/backend/replication/logical/Makefile
+++ b/src/backend/replication/logical/Makefile
@@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(srcdir) $(CPPFLAGS)
OBJS = \
+ applybgworker.o \
decode.o \
launcher.o \
logical.o \
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
new file mode 100644
index 0000000000..dfa49be98c
--- /dev/null
+++ b/src/backend/replication/logical/applybgworker.c
@@ -0,0 +1,775 @@
+/*-------------------------------------------------------------------------
+ * applybgworker.c
+ * Support routines for applying xact by apply background worker
+ *
+ * Copyright (c) 2016-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/logical/applybgworker.c
+ *
+ * This file contains routines that are intended to support setting up, using,
+ * and tearing down a ApplyBgworkerState.
+ *
+ * Refer to the comments in file header of logical/worker.c to see more
+ * information about apply background worker.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "libpq/pqformat.h"
+#include "mb/pg_wchar.h"
+#include "pgstat.h"
+#include "postmaster/interrupt.h"
+#include "replication/logicallauncher.h"
+#include "replication/logicalworker.h"
+#include "replication/origin.h"
+#include "replication/walreceiver.h"
+#include "replication/worker_internal.h"
+#include "storage/ipc.h"
+#include "storage/procarray.h"
+#include "tcop/tcopprot.h"
+#include "utils/inval.h"
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+#include "utils/syscache.h"
+
+#define PG_LOGICAL_APPLY_SHM_MAGIC 0x79fb2447 // TODO Consider change
+
+/*
+ * DSM keys for apply background worker. Unlike other parallel execution code,
+ * since we don't need to worry about DSM keys conflicting with plan_node_id we
+ * can use small integers.
+ */
+#define APPLY_BGWORKER_KEY_SHARED 1
+#define APPLY_BGWORKER_KEY_MQ 2
+
+/* Queue size of DSM, 16 MB for now. */
+#define DSM_QUEUE_SIZE 160000000
+
+/*
+ * There are three fields in message: start_lsn, end_lsn and send_time. Because
+ * we have updated these statistics in apply worker, we could ignore these
+ * fields in apply background worker. (see function LogicalRepApplyLoop)
+ */
+#define IGNORE_SIZE_IN_MESSAGE (3 * sizeof(uint64))
+
+/*
+ * Entry for a hash table we use to map from xid to our apply background worker
+ * state.
+ */
+typedef struct ApplyBgworkerEntry
+{
+ TransactionId xid;
+ ApplyBgworkerState *wstate;
+} ApplyBgworkerEntry;
+
+/* Apply background workers hash table (initialized on first use) */
+static HTAB *ApplyWorkersHash = NULL;
+static List *ApplyWorkersFreeList = NIL;
+static List *ApplyWorkersList = NIL;
+
+/*
+ * Information shared between main apply worker and apply background worker.
+ */
+volatile ApplyBgworkerShared *MyParallelState = NULL;
+
+List *subxactlist = NIL;
+
+static bool apply_bgworker_can_start(TransactionId xid);
+static ApplyBgworkerState *apply_bgworker_setup(void);
+static void apply_bgworker_setup_dsm(ApplyBgworkerState *wstate);
+
+/*
+ * Check if starting a new apply background worker is allowed.
+ */
+static bool
+apply_bgworker_can_start(TransactionId xid)
+{
+ if (!TransactionIdIsValid(xid))
+ return false;
+
+ /*
+ * Don't start a new background worker if not in streaming parallel mode.
+ */
+ if (MySubscription->stream != SUBSTREAM_PARALLEL)
+ return false;
+
+ /*
+ * Don't start a new background worker if user has set skiplsn as it's
+ * possible that user want to skip the streaming transaction. For
+ * streaming transaction, we need to spill the transaction to disk so that
+ * we can get the last LSN of the transaction to judge whether to skip
+ * before starting to apply the change.
+ */
+ if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
+ return false;
+
+ /*
+ * For streaming transactions that are being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time. So, we don't start new apply
+ * background worker in this case.
+ */
+ if (!AllTablesyncsReady())
+ return false;
+
+ return true;
+}
+
+/*
+ * Try to start an apply background worker and, if successful, cache it in
+ * ApplyWorkersHash keyed by the specified xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_start(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerState *wstate;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!apply_bgworker_can_start(xid))
+ return NULL;
+
+ /* First time through, initialize apply workers hashtable */
+ if (ApplyWorkersHash == NULL)
+ {
+ HASHCTL ctl;
+
+ MemSet(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(TransactionId);
+ ctl.entrysize = sizeof(ApplyBgworkerEntry);
+ ctl.hcxt = ApplyContext;
+
+ ApplyWorkersHash = hash_create("logical apply workers hash", 8, &ctl,
+ HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+ }
+
+ /*
+ * Now, we try to get an apply background worker. If there is at least one
+ * worker in the free list, then take one. Otherwise, we try to start a
+ * new apply background worker.
+ */
+ if (list_length(ApplyWorkersFreeList) > 0)
+ {
+ wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
+ ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
+ Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ wstate = apply_bgworker_setup();
+
+ if (wstate == NULL)
+ return NULL;
+ }
+
+ /*
+ * Create entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_ENTER, &found);
+ if (found)
+ elog(ERROR, "hash table corrupted");
+
+ /* Fill up the hash entry */
+ wstate->pstate->status = APPLY_BGWORKER_BUSY;
+ wstate->pstate->server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ wstate->pstate->stream_xid = xid;
+ entry->wstate = wstate;
+ entry->xid = xid;
+
+ return wstate;
+}
+
+/*
+ * Try to look up worker inside ApplyWorkersHash for requested xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_find(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!TransactionIdIsValid(xid))
+ return NULL;
+
+ if (ApplyWorkersHash == NULL)
+ return NULL;
+
+ /*
+ * Find entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
+ if (found)
+ {
+ Assert(entry->wstate->pstate->status == APPLY_BGWORKER_BUSY);
+ return entry->wstate;
+ }
+ else
+ return NULL;
+}
+
+/*
+ * Add the worker to the free list and remove the entry from the hash table.
+ */
+void
+apply_bgworker_free(ApplyBgworkerState *wstate)
+{
+ MemoryContext oldctx;
+ TransactionId xid = wstate->pstate->stream_xid;
+
+ Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ hash_search(ApplyWorkersHash, &xid, HASH_REMOVE, NULL);
+
+ elog(DEBUG1, "adding finished apply worker #%u for xid %u to the free list",
+ wstate->pstate->n, wstate->pstate->stream_xid);
+
+ ApplyWorkersFreeList = lappend(ApplyWorkersFreeList, wstate);
+
+ MemoryContextSwitchTo(oldctx);
+}
+
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *pst)
+{
+ shm_mq_result shmq_res;
+ PGPROC *registrant;
+ ErrorContextCallback errcallback;
+
+ registrant = BackendPidGetProc(MyBgworkerEntry->bgw_notify_pid);
+ SetLatch(®istrant->procLatch);
+
+ /*
+ * Push apply error context callback. Fields will be filled applying a
+ * change.
+ */
+ errcallback.callback = apply_error_callback;
+ errcallback.previous = error_context_stack;
+ error_context_stack = &errcallback;
+
+ for (;;)
+ {
+ void *data;
+ Size len;
+ int c;
+ StringInfoData s;
+ MemoryContext oldctx;
+
+ CHECK_FOR_INTERRUPTS();
+
+ /* Ensure we are reading the data into our memory context. */
+ oldctx = MemoryContextSwitchTo(ApplyMessageContext);
+
+ shmq_res = shm_mq_receive(mqh, &len, &data, false);
+
+ if (shmq_res != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("lost connection to the main apply worker")));
+
+ if (len == 0)
+ break;
+
+ s.cursor = 0;
+ s.maxlen = -1;
+ s.data = (char *) data;
+ s.len = len;
+
+ /*
+ * We use first byte of message for additional communication between
+ * main Logical replication worker and apply background workers, so if
+ * it differs from 'w', then process it first.
+ */
+ c = pq_getmsgbyte(&s);
+ switch (c)
+ {
+ /* End message of streaming chunk */
+ case LOGICAL_REP_MSG_STREAM_STOP:
+ elog(DEBUG1, "[Apply BGW #%u] ended processing streaming chunk,"
+ "waiting on shm_mq_receive", pst->n);
+
+ in_streamed_transaction = false;
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ continue;
+ case 'w':
+ break;
+ default:
+ elog(ERROR, "[Apply BGW #%u] unexpected message \"%c\"",
+ pst->n, c);
+ break;
+ }
+
+ /* Ignore statistics fields that have been updated. */
+ s.cursor += IGNORE_SIZE_IN_MESSAGE;
+
+ apply_dispatch(&s);
+
+ if (ConfigReloadPending)
+ {
+ ConfigReloadPending = false;
+ ProcessConfigFile(PGC_SIGHUP);
+ }
+
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
+ }
+
+ MemoryContextSwitchTo(TopMemoryContext);
+ MemoryContextReset(ApplyContext);
+
+ /* Pop the error context stack */
+ error_context_stack = errcallback.previous;
+
+ elog(DEBUG1, "[Apply BGW #%u] exiting", pst->n);
+
+ /* Signal main process that we are done. */
+ SetLatch(®istrant->procLatch);
+}
+
+/*
+ * Set the exit status so that the main apply worker can realize we have
+ * shutdown.
+ */
+static void
+apply_bgworker_shutdown(int code, Datum arg)
+{
+ SpinLockAcquire(&MyParallelState->mutex);
+ MyParallelState->status = APPLY_BGWORKER_EXIT;
+ SpinLockRelease(&MyParallelState->mutex);
+
+ dsm_detach((dsm_segment *) DatumGetPointer(arg));
+}
+
+/*
+ * Apply Background Worker entry point
+ */
+void
+ApplyBgworkerMain(Datum main_arg)
+{
+ volatile ApplyBgworkerShared *pst;
+
+ dsm_handle handle;
+ dsm_segment *seg;
+ shm_toc *toc;
+ shm_mq *mq;
+ shm_mq_handle *mqh;
+ MemoryContext oldcontext;
+ RepOriginId originid;
+ int worker_slot = DatumGetInt32(main_arg);
+ char originname[NAMEDATALEN];
+
+ MemoryContextSwitchTo(TopMemoryContext);
+
+ /* Init the memory context for the apply background worker to work in. */
+ ApplyContext = AllocSetContextCreate(TopMemoryContext,
+ "ApplyContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Init the ApplyMessageContext which we clean up after each replication
+ * protocol message.
+ */
+ ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+ "ApplyMessageContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /* Setup signal handling */
+ pqsignal(SIGHUP, SignalHandlerForConfigReload);
+ pqsignal(SIGTERM, die);
+ BackgroundWorkerUnblockSignals();
+
+ /*
+ * Connect to the dynamic shared memory segment.
+ *
+ * The backend that registered this worker passed us the ID of a shared
+ * memory segment to which we must attach for further instructions. In
+ * order to attach to dynamic shared memory, we need a resource owner.
+ * Once we've mapped the segment in our address space, attach to the table
+ * of contents so we can locate the various data structures we'll need to
+ * find within the segment.
+ */
+ CurrentResourceOwner = ResourceOwnerCreate(NULL, "Logical apply worker");
+ memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsm_handle));
+ seg = dsm_attach(handle);
+ if (seg == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("unable to map dynamic shared memory segment")));
+ toc = shm_toc_attach(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg));
+ if (toc == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("bad magic number in dynamic shared memory segment")));
+
+ before_shmem_exit(apply_bgworker_shutdown, PointerGetDatum(seg));
+
+ /* Look up the parallel state. */
+ pst = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_SHARED, false);
+ MyParallelState = pst;
+
+ /*
+ * Attach to the message queue.
+ */
+ mq = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_MQ, false);
+ shm_mq_set_receiver(mq, MyProc);
+ mqh = shm_mq_attach(mq, seg, NULL);
+
+ /* Run as replica session replication role. */
+ SetConfigOption("session_replication_role", "replica",
+ PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Now, we have initialized DSM. Attach to slot.
+ */
+ logicalrep_worker_attach(worker_slot);
+
+ /* Connect to our database. */
+ BackgroundWorkerInitializeConnectionByOid(MyLogicalRepWorker->dbid,
+ MyLogicalRepWorker->userid,
+ 0);
+
+ /*
+ * Set always-secure search path, so malicious users can't redirect user
+ * code (e.g. pg_index.indexprs).
+ */
+ SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Set the client encoding to the database encoding, since that is what
+ * the leader will expect.
+ */
+ SetClientEncoding(GetDatabaseEncoding());
+
+ stream_xid = pst->stream_xid;
+
+ StartTransactionCommand();
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ MySubscription = GetSubscription(MyLogicalRepWorker->subid, true);
+ if (!MySubscription)
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply worker for subscription %u will not "
+ "start because the subscription was removed during startup",
+ MyLogicalRepWorker->subid)));
+ proc_exit(0);
+ }
+
+ MySubscriptionValid = true;
+ MemoryContextSwitchTo(oldcontext);
+
+ /* Setup synchronous commit according to the user's wishes */
+ SetConfigOption("synchronous_commit", MySubscription->synccommit,
+ PGC_BACKEND, PGC_S_OVERRIDE);
+
+ /* Keep us informed about subscription changes. */
+ CacheRegisterSyscacheCallback(SUBSCRIPTIONOID,
+ subscription_change_cb,
+ (Datum) 0);
+
+ CommitTransactionCommand();
+
+ /* Setup replication origin tracking. */
+ StartTransactionCommand();
+ snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
+ originid = replorigin_by_name(originname, true);
+ if (!OidIsValid(originid))
+ originid = replorigin_create(originname);
+
+ /*
+ * The apply background worker doesn't need to monopolize this replication
+ * origin which was already acquired by its leader process.
+ */
+ replorigin_session_setup(originid, false);
+ replorigin_session_origin = originid;
+ CommitTransactionCommand();
+
+ /*
+ * Allocate the origin name in long-lived context for error context
+ * message.
+ */
+ apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+ originname);
+
+ elog(DEBUG1, "[Apply BGW #%u] started", pst->n);
+
+ LogicalApplyBgwLoop(mqh, pst);
+
+ /*
+ * We're done. Explicitly detach the shared memory segment so that we
+ * don't get a resource leak warning at commit time. This will fire any
+ * on_dsm_detach callbacks we've registered, as well. Once that's done,
+ * we can go ahead and exit.
+ */
+ dsm_detach(seg);
+ proc_exit(0);
+}
+
+/*
+ * Set up a dynamic shared memory segment.
+ *
+ * We set up a control region that contains a ApplyBgworkerShared,
+ * plus one region per message queue. There are as many message queues as
+ * the number of workers.
+ */
+static void
+apply_bgworker_setup_dsm(ApplyBgworkerState *wstate)
+{
+ shm_toc_estimator e;
+ Size segsize;
+ dsm_segment *seg;
+ shm_toc *toc;
+ ApplyBgworkerShared *pst;
+ shm_mq *mq;
+ int64 queue_size = DSM_QUEUE_SIZE;
+
+ /*
+ * Estimate how much shared memory we need.
+ *
+ * Because the TOC machinery may choose to insert padding of oddly-sized
+ * requests, we must estimate each chunk separately.
+ *
+ * We need one key to register the location of the header, and we need
+ * another key to track the location of the message queue.
+ */
+ shm_toc_initialize_estimator(&e);
+ shm_toc_estimate_chunk(&e, sizeof(ApplyBgworkerShared));
+ shm_toc_estimate_chunk(&e, (Size) queue_size);
+
+ shm_toc_estimate_keys(&e, 1 + 1);
+ segsize = shm_toc_estimate(&e);
+
+ /* Create the shared memory segment and establish a table of contents. */
+ seg = dsm_create(shm_toc_estimate(&e), 0);
+ toc = shm_toc_create(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg),
+ segsize);
+
+ /* Set up the header region. */
+ pst = shm_toc_allocate(toc, sizeof(ApplyBgworkerShared));
+ SpinLockInit(&pst->mutex);
+ pst->status = APPLY_BGWORKER_BUSY;
+ pst->server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ pst->stream_xid = stream_xid;
+ pst->n = list_length(ApplyWorkersList) + 1;
+
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_SHARED, pst);
+
+ /* Set up message queue for the worker. */
+ mq = shm_mq_create(shm_toc_allocate(toc, (Size) queue_size),
+ (Size) queue_size);
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_MQ, mq);
+ shm_mq_set_sender(mq, MyProc);
+
+ /* Attach the queue. */
+ wstate->mq_handle = shm_mq_attach(mq, seg, NULL);
+
+ /* Return results to caller. */
+ wstate->dsm_seg = seg;
+ wstate->pstate = pst;
+}
+
+/*
+ * Start apply background worker process and allocate shared memory for it.
+ */
+static ApplyBgworkerState *
+apply_bgworker_setup(void)
+{
+ MemoryContext oldcontext;
+ bool launched;
+ ApplyBgworkerState *wstate;
+ int napplyworkers;
+
+ elog(DEBUG1, "setting up apply worker #%u", list_length(ApplyWorkersList) + 1);
+
+ /* Check If there are free worker slot(s) */
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+ napplyworkers = logicalrep_apply_bgworker_count(MyLogicalRepWorker->subid);
+ LWLockRelease(LogicalRepWorkerLock);
+ if (napplyworkers >= max_apply_bgworkers_per_subscription)
+ return NULL;
+
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ wstate = (ApplyBgworkerState *) palloc0(sizeof(ApplyBgworkerState));
+
+ /* Setup shared memory */
+ apply_bgworker_setup_dsm(wstate);
+
+ launched = logicalrep_worker_launch(MyLogicalRepWorker->dbid,
+ MySubscription->oid,
+ MySubscription->name,
+ MyLogicalRepWorker->userid,
+ InvalidOid,
+ dsm_segment_handle(wstate->dsm_seg));
+
+ if (launched)
+ ApplyWorkersList = lappend(ApplyWorkersList, wstate);
+ else
+ {
+ dsm_detach(wstate->dsm_seg);
+ wstate->dsm_seg = NULL;
+
+ pfree(wstate);
+ wstate = NULL;
+ }
+
+ MemoryContextSwitchTo(oldcontext);
+
+ return wstate;
+}
+
+/*
+ * Send the data to the specified apply background worker via shared-memory queue.
+ */
+void
+apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes, const void *data)
+{
+ shm_mq_result result;
+
+ result = shm_mq_send(wstate->mq_handle, nbytes, data, false, true);
+
+ if (result != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("could not send tuples to shared-memory queue")));
+}
+
+/*
+ * Wait until the status of apply background worker reaches the
+ * 'wait_for_status'
+ */
+void
+apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status)
+{
+ for (;;)
+ {
+ char status;
+
+ SpinLockAcquire(&wstate->pstate->mutex);
+ status = wstate->pstate->status;
+ SpinLockRelease(&wstate->pstate->mutex);
+
+ /* Done if already in correct status. */
+ if (status == wait_for_status)
+ break;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ wstate->pstate->n, wstate->pstate->stream_xid)));
+
+ /* Wait to be signalled. */
+ WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 0,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE);
+
+ /* Reset the latch so we don't spin. */
+ ResetLatch(MyLatch);
+
+ /* An interrupt may have occurred while we were waiting. */
+ CHECK_FOR_INTERRUPTS();
+ }
+}
+
+/*
+ * Check the status of workers and report an error if any apply background
+ * worker has exited unexpectedly.
+ */
+void
+apply_bgworker_check_status(void)
+{
+ ListCell *lc;
+
+ if (am_apply_bgworker() || MySubscription->stream != SUBSTREAM_PARALLEL)
+ return;
+
+ foreach(lc, ApplyWorkersList)
+ {
+ ApplyBgworkerState *wstate = (ApplyBgworkerState *) lfirst(lc);
+
+ /*
+ * We don't lock here as in the worst case we will just detect the
+ * failure of worker a bit later.
+ */
+ if (wstate->pstate->status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u exited unexpectedly",
+ wstate->pstate->n)));
+ }
+
+ /*
+ * Exit if any relation is not in the READY state and if any worker is
+ * handling the streaming transaction at the same time. Because for
+ * streaming transactions that is being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time.
+ */
+ if (list_length(ApplyWorkersFreeList) != list_length(ApplyWorkersList) &&
+ !AllTablesyncsReady())
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply workers for subscription \"%s\" will restart",
+ MySubscription->name),
+ errdetail("Cannot handle streamed replication transaction by apply "
+ "background workers until all tables are synchronized")));
+
+ proc_exit(0);
+ }
+}
+
+/* Set the apply background worker status */
+void
+apply_bgworker_set_status(ApplyBgworkerStatus status)
+{
+ if (!am_apply_bgworker())
+ return;
+
+ elog(DEBUG1, "[Apply BGW #%u] set status to %d", MyParallelState->n, status);
+
+ SpinLockAcquire(&MyParallelState->mutex);
+ MyParallelState->status = status;
+ SpinLockRelease(&MyParallelState->mutex);
+}
+
+/*
+ * Define a savepoint for a subxact in apply background worker if needed.
+ *
+ * Inside apply background worker we can figure out that new subtransaction was
+ * started if new change arrived with different xid. In that case we can define
+ * named savepoint, so that we were able to commit/rollback it separately
+ * later.
+ * Special case is if the first change comes from subtransaction, then
+ * we check that current_xid differs from stream_xid.
+ */
+void
+apply_bgworker_subxact_info_add(TransactionId current_xid)
+{
+ if (current_xid != stream_xid &&
+ !list_member_int(subxactlist, (int) current_xid))
+ {
+ MemoryContext oldctx;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", current_xid);
+
+ elog(DEBUG1, "[Apply BGW #%u] defining savepoint %s",
+ MyParallelState->n, spname);
+
+ DefineSavepoint(spname);
+ CommitTransactionCommand();
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+ subxactlist = lappend_int(subxactlist, (int) current_xid);
+ MemoryContextSwitchTo(oldctx);
+ }
+}
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c5c6a2ba68..d4d5093a0b 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -651,9 +651,10 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
{
for (i = 0; i < parsed->nsubxacts; i++)
{
- ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr);
+ ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr,
+ commit_time);
}
- ReorderBufferForget(ctx->reorder, xid, buf->origptr);
+ ReorderBufferForget(ctx->reorder, xid, buf->origptr, commit_time);
return;
}
@@ -821,10 +822,11 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
for (i = 0; i < parsed->nsubxacts; i++)
{
ReorderBufferAbort(ctx->reorder, parsed->subxacts[i],
- buf->record->EndRecPtr);
+ buf->record->EndRecPtr, abort_time);
}
- ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr);
+ ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr,
+ abort_time);
}
/* update the decoding stats */
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 2bdab53e19..d11d2361c6 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -54,6 +54,7 @@
int max_logical_replication_workers = 4;
int max_sync_workers_per_subscription = 2;
+int max_apply_bgworkers_per_subscription = 2;
LogicalRepWorker *MyLogicalRepWorker = NULL;
@@ -73,6 +74,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg);
static void logicalrep_worker_onexit(int code, Datum arg);
static void logicalrep_worker_detach(void);
static void logicalrep_worker_cleanup(LogicalRepWorker *worker);
+static void logicalrep_worker_stop_internal(LogicalRepWorker *worker);
static bool on_commit_launcher_wakeup = false;
@@ -151,8 +153,10 @@ get_subscription_list(void)
*
* This is only needed for cleaning up the shared memory in case the worker
* fails to attach.
+ *
+ * Return false if the attach fails. Otherwise return true.
*/
-static void
+static bool
WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
uint16 generation,
BackgroundWorkerHandle *handle)
@@ -168,11 +172,11 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
- /* Worker either died or has started; no need to do anything. */
+ /* Worker either died or has started. Return false if died. */
if (!worker->in_use || worker->proc)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return worker->in_use;
}
LWLockRelease(LogicalRepWorkerLock);
@@ -187,7 +191,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
if (generation == worker->generation)
logicalrep_worker_cleanup(worker);
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -223,6 +227,13 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running)
{
LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+ /*
+ * We are only interested in the main apply worker or table sync worker
+ * here.
+ */
+ if (w->subworker)
+ continue;
+
if (w->in_use && w->subid == subid && w->relid == relid &&
(!only_running || w->proc))
{
@@ -259,11 +270,11 @@ logicalrep_workers_find(Oid subid, bool only_running)
}
/*
- * Start new apply background worker, if possible.
+ * Start new background worker, if possible.
*/
-void
+bool
logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
- Oid relid)
+ Oid relid, dsm_handle subworker_dsm)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
@@ -273,6 +284,10 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
LogicalRepWorker *worker = NULL;
int nsyncworkers;
TimestampTz now;
+ bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
+
+ /* Sanity check : we don't support table sync in subworker. */
+ Assert(!(is_subworker && OidIsValid(relid)));
ereport(DEBUG1,
(errmsg_internal("starting logical replication worker for subscription \"%s\"",
@@ -350,7 +365,7 @@ retry:
if (OidIsValid(relid) && nsyncworkers >= max_sync_workers_per_subscription)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -364,7 +379,7 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of logical replication worker slots"),
errhint("You might need to increase max_logical_replication_workers.")));
- return;
+ return false;
}
/* Prepare the worker slot. */
@@ -379,6 +394,7 @@ retry:
worker->relstate = SUBREL_STATE_UNKNOWN;
worker->relstate_lsn = InvalidXLogRecPtr;
worker->stream_fileset = NULL;
+ worker->subworker = is_subworker;
worker->last_lsn = InvalidXLogRecPtr;
TIMESTAMP_NOBEGIN(worker->last_send_time);
TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -396,19 +412,31 @@ retry:
BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
- snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
+ if (is_subworker)
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyBgworkerMain");
+ else
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
if (OidIsValid(relid))
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u sync %u", subid, relid);
+ else if (is_subworker)
+ snprintf(bgw.bgw_name, BGW_MAXLEN,
+ "logical replication apply background worker for subscription %u", subid);
else
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u", subid);
+
snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = Int32GetDatum(slot);
+ if (is_subworker)
+ memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle));
+
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
/* Failed to start worker, so clean up the worker slot. */
@@ -421,11 +449,11 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of background worker slots"),
errhint("You might need to increase max_worker_processes.")));
- return;
+ return false;
}
/* Now wait until it attaches. */
- WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
+ return WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
}
/*
@@ -436,18 +464,27 @@ void
logicalrep_worker_stop(Oid subid, Oid relid)
{
LogicalRepWorker *worker;
- uint16 generation;
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
worker = logicalrep_worker_find(subid, relid, false);
- /* No worker, nothing to do. */
- if (!worker)
- {
- LWLockRelease(LogicalRepWorkerLock);
- return;
- }
+ if (worker)
+ logicalrep_worker_stop_internal(worker);
+
+ LWLockRelease(LogicalRepWorkerLock);
+}
+
+/*
+ * Workhorse for logicalrep_worker_stop() and logicalrep_worker_detach(). Stop
+ * the worker and wait for it to die.
+ */
+static void
+logicalrep_worker_stop_internal(LogicalRepWorker *worker)
+{
+ uint16 generation;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
/*
* Remember which generation was our worker so we can check if what we see
@@ -485,10 +522,7 @@ logicalrep_worker_stop(Oid subid, Oid relid)
* different, meaning that a different worker has taken the slot.
*/
if (!worker->in_use || worker->generation != generation)
- {
- LWLockRelease(LogicalRepWorkerLock);
return;
- }
/* Worker has assigned proc, so it has started. */
if (worker->proc)
@@ -522,8 +556,6 @@ logicalrep_worker_stop(Oid subid, Oid relid)
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
}
-
- LWLockRelease(LogicalRepWorkerLock);
}
/*
@@ -599,6 +631,29 @@ logicalrep_worker_attach(int slot)
static void
logicalrep_worker_detach(void)
{
+ /*
+ * If we are the main apply worker, stop all the apply background workers
+ * we started before.
+ */
+ if (!MyLogicalRepWorker->subworker)
+ {
+ List *workers;
+ ListCell *lc;
+
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+
+ workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true);
+ foreach(lc, workers)
+ {
+ LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc);
+
+ if (w->subworker)
+ logicalrep_worker_stop_internal(w);
+ }
+
+ LWLockRelease(LogicalRepWorkerLock);
+ }
+
/* Block concurrent access. */
LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
@@ -621,6 +676,7 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker)
worker->userid = InvalidOid;
worker->subid = InvalidOid;
worker->relid = InvalidOid;
+ worker->subworker = false;
}
/*
@@ -679,6 +735,30 @@ logicalrep_sync_worker_count(Oid subid)
return res;
}
+/*
+ * Count the number of registered (not necessarily running) apply background
+ * workers for a subscription.
+ */
+int
+logicalrep_apply_bgworker_count(Oid subid)
+{
+ int i;
+ int res = 0;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
+
+ /* Search for attached worker for a given subscription id. */
+ for (i = 0; i < max_logical_replication_workers; i++)
+ {
+ LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+
+ if (w->subid == subid && w->subworker)
+ res++;
+ }
+
+ return res;
+}
+
/*
* ApplyLauncherShmemSize
* Compute space needed for replication launcher shared memory
@@ -868,7 +948,7 @@ ApplyLauncherMain(Datum main_arg)
wait_time = wal_retrieve_retry_interval;
logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
- sub->owner, InvalidOid);
+ sub->owner, InvalidOid, DSM_HANDLE_INVALID);
}
}
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index 21937ab2d3..50c567fb6e 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1063,12 +1063,21 @@ ReplicationOriginExitCleanup(int code, Datum arg)
* array doesn't have to be searched when calling
* replorigin_session_advance().
*
- * Obviously only one such cached origin can exist per process and the current
+ * Normally only one such cached origin can exist per process and the current
* cached value can only be set again after the previous value is torn down
* with replorigin_session_reset().
+ *
+ * However, if the function parameter 'must_acquire' is false, we allow the
+ * process to use the same slot already acquired by another process. It's safe
+ * because 1) The only caller (apply background workers) will maintain the
+ * commit order by allowing only one process to commit at a time, so no two
+ * workers will be operating on the same origin at the same time (see comments
+ * in logical/worker.c). 2) Even though we try to advance the session's origin
+ * concurrently, it's safe to do so as we change/advance the session_origin
+ * LSNs under replicate_state LWLock.
*/
void
-replorigin_session_setup(RepOriginId node)
+replorigin_session_setup(RepOriginId node, bool must_acquire)
{
static bool registered_cleanup;
int i;
@@ -1110,7 +1119,7 @@ replorigin_session_setup(RepOriginId node)
if (curstate->roident != node)
continue;
- else if (curstate->acquired_by != 0)
+ else if (curstate->acquired_by != 0 && must_acquire)
{
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
@@ -1141,7 +1150,14 @@ replorigin_session_setup(RepOriginId node)
Assert(session_replication_state->roident != InvalidRepOriginId);
- session_replication_state->acquired_by = MyProcPid;
+ if (must_acquire)
+ session_replication_state->acquired_by = MyProcPid;
+ else if (session_replication_state->acquired_by == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+ errmsg("apply background worker could not find replication state slot for replication origin with OID %u",
+ node),
+ errdetail("There is no replication state slot set by its main apply worker.")));
LWLockRelease(ReplicationOriginLock);
@@ -1321,7 +1337,7 @@ pg_replication_origin_session_setup(PG_FUNCTION_ARGS)
name = text_to_cstring((text *) DatumGetPointer(PG_GETARG_DATUM(0)));
origin = replorigin_by_name(name, false);
- replorigin_session_setup(origin);
+ replorigin_session_setup(origin, true);
replorigin_session_origin = origin;
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index ff8513e2d2..affd08cfa4 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -1163,31 +1163,56 @@ logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data)
/*
* Write STREAM ABORT to the output stream. Note that xid and subxid will be
* same for the top-level transaction abort.
+ *
+ * If write_abort_lsn is true, send the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
void
logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid)
+ ReorderBufferTXN *txn, XLogRecPtr abort_lsn,
+ bool write_abort_lsn)
{
pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT);
- Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid));
+ Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(txn->xid));
/* transaction ID */
pq_sendint32(out, xid);
- pq_sendint32(out, subxid);
+ pq_sendint32(out, txn->xid);
+
+ if (write_abort_lsn)
+ {
+ pq_sendint64(out, abort_lsn);
+ pq_sendint64(out, txn->xact_time.abort_time);
+ }
}
/*
* Read STREAM ABORT from the output stream.
+ *
+ * If read_abort_lsn is true, try to read the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
void
-logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid)
+logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn)
{
- Assert(xid && subxid);
+ Assert(abort_data);
- *xid = pq_getmsgint(in, 4);
- *subxid = pq_getmsgint(in, 4);
+ abort_data->xid = pq_getmsgint(in, 4);
+ abort_data->subxid = pq_getmsgint(in, 4);
+
+ if (read_abort_lsn)
+ {
+ abort_data->abort_lsn = pq_getmsgint64(in);
+ abort_data->abort_time = pq_getmsgint64(in);
+ }
+ else
+ {
+ abort_data->abort_lsn = InvalidXLogRecPtr;
+ abort_data->abort_time = 0;
+ }
}
/*
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 88a37fde72..8989328046 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -2826,7 +2826,8 @@ ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
* disk.
*/
void
-ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2837,6 +2838,8 @@ ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
{
@@ -2911,7 +2914,8 @@ ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid)
* to this xid might re-create the transaction incompletely.
*/
void
-ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2922,6 +2926,8 @@ ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
rb->stream_abort(rb, txn, lsn);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 670c6fcada..8ffba7e2e5 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -568,7 +568,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
MySubscription->oid,
MySubscription->name,
MyLogicalRepWorker->userid,
- rstate->relid);
+ rstate->relid,
+ DSM_HANDLE_INVALID);
hentry->last_start_time = now;
}
}
@@ -589,6 +590,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
void
process_syncing_tables(XLogRecPtr current_lsn)
{
+ if (MyLogicalRepWorker->subworker)
+ return;
+
if (am_tablesync_worker())
process_syncing_tables_for_sync(current_lsn);
else
@@ -1273,7 +1277,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
* time this tablesync was launched.
*/
originid = replorigin_by_name(originname, false);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
*origin_startpos = replorigin_session_get_progress(false);
@@ -1384,7 +1388,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
true /* go backward */ , true /* WAL log */ );
UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
}
else
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 38e3b1c1b3..9da11dd41c 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -22,8 +22,28 @@
* STREAMED TRANSACTIONS
* ---------------------
* Streamed transactions (large transactions exceeding a memory limit on the
- * upstream) are not applied immediately, but instead, the data is written
- * to temporary files and then applied at once when the final commit arrives.
+ * upstream) are applied using one of two approaches.
+ *
+ * 1) Separate background workers
+ *
+ * If streaming = parallel, We assign a new apply background worker (if
+ * available) as soon as the xact's first stream is received. The main apply
+ * worker will send changes to this new worker via shared memory. We keep this
+ * worker assigned till the transaction commit is received and also wait for
+ * the worker to finish at commit. This preserves commit ordering and avoids
+ * file I/O in most cases. We still need to spill to a file if there is no
+ * worker available. It is important to maintain commit order to avoid failures
+ * due to (a) transaction dependencies, say if we insert a row in the first
+ * transaction and update it in the second transaction then allowing to apply
+ * both in parallel can lead to failure in the update. (b) deadlocks, allowing
+ * transactions that update the same set of rows/tables in opposite order to be
+ * applied in parallel can lead to deadlocks.
+ *
+ * 2) Write to temporary files and apply when the final commit arrives
+ *
+ * If no worker is available to handle streamed transaction, the data is
+ * written to temporary files and then applied at once when the final commit
+ * arrives.
*
* Unlike the regular (non-streamed) case, handling streamed transactions has
* to handle aborts of both the toplevel transaction and subtransactions. This
@@ -219,20 +239,8 @@ typedef struct ApplyExecutionData
PartitionTupleRouting *proute; /* partition routing info */
} ApplyExecutionData;
-/* Struct for saving and restoring apply errcontext information */
-typedef struct ApplyErrorCallbackArg
-{
- LogicalRepMsgType command; /* 0 if invalid */
- LogicalRepRelMapEntry *rel;
-
- /* Remote node information */
- int remote_attnum; /* -1 if invalid */
- TransactionId remote_xid;
- XLogRecPtr finish_lsn;
- char *origin_name;
-} ApplyErrorCallbackArg;
-
-static ApplyErrorCallbackArg apply_error_callback_arg =
+/* errcontext tracker */
+ApplyErrorCallbackArg apply_error_callback_arg =
{
.command = 0,
.rel = NULL,
@@ -242,7 +250,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg =
.origin_name = NULL,
};
-static MemoryContext ApplyMessageContext = NULL;
+MemoryContext ApplyMessageContext = NULL;
MemoryContext ApplyContext = NULL;
/* per stream context for streaming transactions */
@@ -251,27 +259,38 @@ static MemoryContext LogicalStreamingContext = NULL;
WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
Subscription *MySubscription = NULL;
-static bool MySubscriptionValid = false;
+bool MySubscriptionValid = false;
bool in_remote_transaction = false;
static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
/* fields valid only when processing streamed transaction */
-static bool in_streamed_transaction = false;
+bool in_streamed_transaction = false;
+
+TransactionId stream_xid = InvalidTransactionId;
+static ApplyBgworkerState *stream_apply_worker = NULL;
-static TransactionId stream_xid = InvalidTransactionId;
+/* Check if we are applying the transaction in an apply background worker */
+#define apply_bgworker_active() (in_streamed_transaction && stream_apply_worker != NULL)
+
+/*
+ * The number of changes during one streaming block (only for apply background
+ * workers)
+ */
+static uint32 nchanges = 0;
/*
* We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
* the subscription if the remote transaction's finish LSN matches the subskiplsn.
* Once we start skipping changes, we don't stop it until we skip all changes of
* the transaction even if pg_subscription is updated and MySubscription->skiplsn
- * gets changed or reset during that. Also, in streaming transaction cases, we
- * don't skip receiving and spooling the changes since we decide whether or not
+ * gets changed or reset during that. Also, in streaming transaction cases (streaming = on),
+ * we don't skip receiving and spooling the changes since we decide whether or not
* to skip applying the changes when starting to apply changes. The subskiplsn is
* cleared after successfully skipping the transaction or applying non-empty
* transaction. The latter prevents the mistakenly specified subskiplsn from
- * being left.
+ * being left. Note that we cannot skip the streaming transaction in parallel
+ * mode, because we cannot get the finish LSN before applying the changes.
*/
static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr;
#define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn)))
@@ -324,9 +343,6 @@ static void maybe_reread_subscription(void);
static void DisableSubscriptionAndExit(void);
-/* prototype needed because of stream_commit */
-static void apply_dispatch(StringInfo s);
-
static void apply_handle_commit_internal(LogicalRepCommitData *commit_data);
static void apply_handle_insert_internal(ApplyExecutionData *edata,
ResultRelInfo *relinfo,
@@ -359,7 +375,6 @@ static void stop_skipping_changes(void);
static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
/* Functions for apply error callback */
-static void apply_error_callback(void *arg);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
@@ -426,40 +441,85 @@ end_replication_step(void)
}
/*
- * Handle streamed transactions.
+ * Handle streamed transactions for both the main apply worker and the apply
+ * background workers.
+ *
+ * In streaming case (receiving a block of streamed transaction), for
+ * SUBSTREAM_ON mode, we simply redirect it to a file for the proper toplevel
+ * transaction, and for SUBSTREAM_PARALLEL mode, we send the changes to
+ * apply background worker (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE
+ * changes will also be applied in main apply worker).
*
- * If in streaming mode (receiving a block of streamed transaction), we
- * simply redirect it to a file for the proper toplevel transaction.
+ * For non-streamed transactions, returns false;
+ * For streamed transactions, returns true if in main apply worker, false
+ * otherwise.
*
- * Returns true for streamed transactions, false otherwise (regular mode).
+ * Exception: When parallel mode is applying streamed transaction in the main
+ * apply worker, (e.g. when addressing LOGICAL_REP_MSG_RELATION or
+ * LOGICAL_REP_MSG_TYPE changes), then return false.
*/
static bool
handle_streamed_transaction(LogicalRepMsgType action, StringInfo s)
{
- TransactionId xid;
+ TransactionId current_xid = InvalidTransactionId;
- /* not in streaming mode */
- if (!in_streamed_transaction)
+ /* Not in streaming mode */
+ if (!(in_streamed_transaction || am_apply_bgworker()))
return false;
- Assert(stream_fd != NULL);
Assert(TransactionIdIsValid(stream_xid));
/*
* We should have received XID of the subxact as the first part of the
* message, so extract it.
*/
- xid = pq_getmsgint(s, 4);
+ current_xid = pq_getmsgint(s, 4);
- if (!TransactionIdIsValid(xid))
+ if (!TransactionIdIsValid(current_xid))
ereport(ERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("invalid transaction ID in streamed replication transaction")));
- /* Add the new subxact to the array (unless already there). */
- subxact_info_add(xid);
+ if (am_apply_bgworker())
+ {
+ /* Define a savepoint for a subxact if needed. */
+ apply_bgworker_subxact_info_add(current_xid);
+
+ return false;
+ }
+
+ if (apply_bgworker_active())
+ {
+ /*
+ * This is the main apply worker, but there is an apply background
+ * worker, so apply the changes of this transaction in that background
+ * worker. Pass the data to the worker.
+ */
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
+ nchanges += 1;
+
+ /*
+ * XXX The publisher side doesn't always send relation/type update
+ * messages after the streaming transaction, so also update the
+ * relation/type in main apply worker here. See function
+ * cleanup_rel_sync_cache.
+ */
+ if (action == LOGICAL_REP_MSG_RELATION ||
+ action == LOGICAL_REP_MSG_TYPE)
+ return false;
+
+ return true;
+ }
+
+ /*
+ * This is the main apply worker, but there is no apply background worker,
+ * so write to temporary files and apply when the final commit arrives.
+ *
+ * Add the new subxact to the array (unless already there).
+ */
+ subxact_info_add(current_xid);
- /* write the change to the current file */
+ /* Write the change to the current file */
stream_write_change(action, s);
return true;
@@ -844,6 +904,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -898,7 +961,9 @@ apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data)
* BeginTransactionBlock is necessary to balance the EndTransactionBlock
* called within the PrepareTransactionBlock below.
*/
- BeginTransactionBlock();
+ if (!IsTransactionBlock())
+ BeginTransactionBlock();
+
CommitTransactionCommand(); /* Completes the preceding Begin command. */
/*
@@ -950,6 +1015,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1064,10 +1132,6 @@ apply_handle_rollback_prepared(StringInfo s)
/*
* Handle STREAM PREPARE.
- *
- * Logic is in two parts:
- * 1. Replay all the spooled operations
- * 2. Mark the transaction as prepared
*/
static void
apply_handle_stream_prepare(StringInfo s)
@@ -1088,24 +1152,76 @@ apply_handle_stream_prepare(StringInfo s)
logicalrep_read_stream_prepare(s, &prepare_data);
set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn);
- elog(DEBUG1, "received prepare for streamed transaction %u", prepare_data.xid);
+ if (am_apply_bgworker())
+ {
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
- /* Replay all the spooled operations. */
- apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+ CommitTransactionCommand();
- /* Mark the transaction as prepared. */
- apply_handle_prepare_internal(&prepare_data);
+ pgstat_report_stat(false);
- CommitTransactionCommand();
+ list_free(subxactlist);
+ subxactlist = NIL;
- pgstat_report_stat(false);
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(prepare_data.xid);
- store_flush_position(prepare_data.end_lsn);
+ elog(DEBUG1, "received prepare for streamed transaction %u",
+ prepare_data.xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+
+ pgstat_report_stat(false);
+ store_flush_position(prepare_data.end_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
+
+ CommitTransactionCommand();
+
+ pgstat_report_stat(false);
+
+ store_flush_position(prepare_data.end_lsn);
+
+ in_remote_transaction = false;
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ }
+ }
in_remote_transaction = false;
+ stream_apply_worker = NULL;
- /* unlink the files with serialized changes and subxact info. */
- stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1155,15 +1271,6 @@ apply_handle_stream_start(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("duplicate STREAM START message")));
- /*
- * Start a transaction on stream start, this transaction will be committed
- * on the stream stop unless it is a tablesync worker in which case it
- * will be committed after processing all the messages. We need the
- * transaction for handling the buffile, used for serializing the
- * streaming data and subxact info.
- */
- begin_replication_step();
-
/* notify handle methods we're processing a remote transaction */
in_streamed_transaction = true;
@@ -1177,36 +1284,93 @@ apply_handle_stream_start(StringInfo s)
set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr);
- /*
- * Initialize the worker's stream_fileset if we haven't yet. This will be
- * used for the entire duration of the worker so create it in a permanent
- * context. We create this on the very first streaming message from any
- * transaction and then use it for this and other streaming transactions.
- * Now, we could create a fileset at the start of the worker as well but
- * then we won't be sure that it will ever be used.
- */
- if (MyLogicalRepWorker->stream_fileset == NULL)
+ if (am_apply_bgworker())
{
- MemoryContext oldctx;
-
- oldctx = MemoryContextSwitchTo(ApplyContext);
+ /*
+ * Make sure the handle apply_dispatch methods are aware we're in a
+ * remote transaction.
+ */
+ in_remote_transaction = true;
- MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
- FileSetInit(MyLogicalRepWorker->stream_fileset);
+ /* Begin the transaction. */
+ AcceptInvalidationMessages();
+ maybe_reread_subscription();
- MemoryContextSwitchTo(oldctx);
+ StartTransactionCommand();
+ BeginTransactionBlock();
+ CommitTransactionCommand();
}
+ else
+ {
+ /*
+ * This is the main apply worker. Check if there is any free apply
+ * background worker we can use to process this transaction.
+ */
+ if (first_segment)
+ stream_apply_worker = apply_bgworker_start(stream_xid);
+ else
+ stream_apply_worker = apply_bgworker_find(stream_xid);
- /* open the spool file for this transaction */
- stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+ if (stream_apply_worker)
+ {
+ /*
+ * If we have found a free worker or if we are already applying this
+ * transaction in an apply background worker, then we pass the data to
+ * that worker.
+ */
+ if (first_segment)
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
- /* if this is not the first segment, open existing subxact file */
- if (!first_segment)
- subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+ nchanges = 0;
+ elog(DEBUG1, "starting streaming of xid %u", stream_xid);
+ }
+ else
+ {
+ /*
+ * Since no apply background worker is available for the first
+ * stream start, serialize all the changes of the transaction.
+ *
+ * Start a transaction on stream start, this transaction will be
+ * committed on the stream stop unless it is a tablesync worker in
+ * which case it will be committed after processing all the
+ * messages. We need the transaction for handling the buffile,
+ * used for serializing the streaming data and subxact info.
+ */
+ begin_replication_step();
- pgstat_report_activity(STATE_RUNNING, NULL);
+ /*
+ * Initialize the worker's stream_fileset if we haven't yet. This will
+ * be used for the entire duration of the worker so create it in a
+ * permanent context. We create this on the very first streaming
+ * message from any transaction and then use it for this and other
+ * streaming transactions. Now, we could create a fileset at the start
+ * of the worker as well but then we won't be sure that it will ever
+ * be used.
+ */
+ if (MyLogicalRepWorker->stream_fileset == NULL)
+ {
+ MemoryContext oldctx;
- end_replication_step();
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
+ FileSetInit(MyLogicalRepWorker->stream_fileset);
+
+ MemoryContextSwitchTo(oldctx);
+ }
+
+ /* Open the spool file for this transaction. */
+ stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+
+ /* If this is not the first segment, open existing subxact file. */
+ if (!first_segment)
+ subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+
+ end_replication_step();
+ }
+ }
+
+ pgstat_report_activity(STATE_RUNNING, NULL);
}
/*
@@ -1220,53 +1384,52 @@ apply_handle_stream_stop(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("STREAM STOP message without STREAM START")));
- /*
- * Close the file with serialized changes, and serialize information about
- * subxacts for the toplevel transaction.
- */
- subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
- stream_close_file();
+ if (apply_bgworker_active())
+ {
+ char action = LOGICAL_REP_MSG_STREAM_STOP;
- /* We must be in a valid transaction state */
- Assert(IsTransactionState());
+ apply_bgworker_send_data(stream_apply_worker, 1, &action);
- /* Commit the per-stream transaction */
- CommitTransactionCommand();
+ elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
+ }
+ else
+ {
+ /*
+ * Close the file with serialized changes, and serialize information
+ * about subxacts for the toplevel transaction.
+ */
+ subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
+ stream_close_file();
- in_streamed_transaction = false;
+ /* We must be in a valid transaction state */
+ Assert(IsTransactionState());
- /* Reset per-stream context */
- MemoryContextReset(LogicalStreamingContext);
+ /* Commit the per-stream transaction */
+ CommitTransactionCommand();
+
+ /* Reset per-stream context */
+ MemoryContextReset(LogicalStreamingContext);
+ }
+
+ in_streamed_transaction = false;
+ stream_apply_worker = NULL;
pgstat_report_activity(STATE_IDLE, NULL);
reset_apply_error_context_info();
}
/*
- * Handle STREAM abort message.
+ * Handle STREAM ABORT message when the transaction was spilled to disk.
*/
static void
-apply_handle_stream_abort(StringInfo s)
+serialize_stream_abort(TransactionId xid, TransactionId subxid)
{
- TransactionId xid;
- TransactionId subxid;
-
- if (in_streamed_transaction)
- ereport(ERROR,
- (errcode(ERRCODE_PROTOCOL_VIOLATION),
- errmsg_internal("STREAM ABORT message without STREAM STOP")));
-
- logicalrep_read_stream_abort(s, &xid, &subxid);
-
/*
* If the two XIDs are the same, it's in fact abort of toplevel xact, so
* just delete the files with serialized info.
*/
if (xid == subxid)
- {
- set_apply_error_context_xact(xid, InvalidXLogRecPtr);
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
- }
else
{
/*
@@ -1290,8 +1453,6 @@ apply_handle_stream_abort(StringInfo s)
bool found = false;
char path[MAXPGPATH];
- set_apply_error_context_xact(subxid, InvalidXLogRecPtr);
-
subidx = -1;
begin_replication_step();
subxact_info_read(MyLogicalRepWorker->subid, xid);
@@ -1316,7 +1477,6 @@ apply_handle_stream_abort(StringInfo s)
cleanup_subxact_info();
end_replication_step();
CommitTransactionCommand();
- reset_apply_error_context_info();
return;
}
@@ -1339,6 +1499,142 @@ apply_handle_stream_abort(StringInfo s)
end_replication_step();
CommitTransactionCommand();
}
+}
+
+/*
+ * Handle STREAM ABORT message.
+ */
+static void
+apply_handle_stream_abort(StringInfo s)
+{
+ TransactionId xid;
+ TransactionId subxid;
+ LogicalRepStreamAbortData abort_data;
+ bool read_abort_lsn = false;
+
+ if (in_streamed_transaction)
+ ereport(ERROR,
+ (errcode(ERRCODE_PROTOCOL_VIOLATION),
+ errmsg_internal("STREAM ABORT message without STREAM STOP")));
+
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelState->server_version >= 160000;
+
+ logicalrep_read_stream_abort(s, &abort_data, read_abort_lsn);
+
+ xid = abort_data.xid;
+ subxid = abort_data.subxid;
+
+ set_apply_error_context_xact(subxid, abort_data.abort_lsn);
+
+ if (am_apply_bgworker())
+ {
+ elog(DEBUG1, "[Apply BGW #%u] aborting current transaction xid=%u, subxid=%u",
+ MyParallelState->n, GetCurrentTransactionIdIfAny(),
+ GetCurrentSubTransactionId());
+
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ if (read_abort_lsn)
+ {
+ replorigin_session_origin_lsn = abort_data.abort_lsn;
+ replorigin_session_origin_timestamp = abort_data.abort_time;
+ }
+
+ /*
+ * If the two XIDs are the same, it's in fact abort of toplevel xact,
+ * so just free the subxactlist.
+ */
+ if (subxid == xid)
+ {
+ AbortCurrentTransaction();
+
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
+
+ in_remote_transaction = false;
+ pgstat_report_activity(STATE_IDLE, NULL);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /*
+ * OK, so it's a subxact. Rollback to the savepoint.
+ *
+ * We also need to read the subxactlist, determine the offset
+ * tracked for the subxact, and truncate the list.
+ */
+ int i;
+ bool found = false;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", subxid);
+
+ elog(DEBUG1, "[Apply BGW #%u] rolling back to savepoint %s",
+ MyParallelState->n, spname);
+
+ for (i = list_length(subxactlist) - 1; i >= 0; i--)
+ {
+ xid = (TransactionId) list_nth_int(subxactlist, i);
+ if (xid == subxid)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if (found)
+ {
+ RollbackToSavepoint(spname);
+ CommitTransactionCommand();
+ subxactlist = list_truncate(subxactlist, i + 1);
+ }
+
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ }
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM ABORT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ if (subxid == xid)
+ {
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+ }
+ }
+ else
+ {
+ /*
+ * We are in main apply worker and the transaction has been
+ * serialized to file.
+ */
+ serialize_stream_abort(xid, subxid);
+ }
+ }
reset_apply_error_context_info();
}
@@ -1468,8 +1764,8 @@ apply_spooled_messages(TransactionId xid, XLogRecPtr lsn)
static void
apply_handle_stream_commit(StringInfo s)
{
- TransactionId xid;
LogicalRepCommitData commit_data;
+ TransactionId xid;
if (in_streamed_transaction)
ereport(ERROR,
@@ -1479,14 +1775,81 @@ apply_handle_stream_commit(StringInfo s)
xid = logicalrep_read_stream_commit(s, &commit_data);
set_apply_error_context_xact(xid, commit_data.commit_lsn);
- elog(DEBUG1, "received commit for streamed transaction %u", xid);
+ if (am_apply_bgworker())
+ {
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ replorigin_session_origin_lsn = commit_data.end_lsn;
+ replorigin_session_origin_timestamp = commit_data.committime;
+
+ CommitTransactionCommand();
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
- apply_spooled_messages(xid, commit_data.commit_lsn);
+ in_remote_transaction = false;
- apply_handle_commit_internal(&commit_data);
+ pgstat_report_stat(false);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ elog(DEBUG1, "received commit for streamed transaction %u", xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM COMMIT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
- /* unlink the files with serialized changes and subxact info */
- stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ pgstat_report_stat(false);
+ store_flush_position(commit_data.end_lsn);
+ stop_skipping_changes();
+
+ apply_bgworker_free(wstate);
+
+ /*
+ * The transaction is either non-empty or skipped, so we clear the
+ * subskiplsn.
+ */
+ clear_subscription_skip_lsn(commit_data.commit_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(xid, commit_data.commit_lsn);
+
+ apply_handle_commit_internal(&commit_data);
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ }
+ }
+
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -2467,7 +2830,7 @@ apply_handle_truncate(StringInfo s)
/*
* Logical replication protocol message dispatcher.
*/
-static void
+void
apply_dispatch(StringInfo s)
{
LogicalRepMsgType action = pq_getmsgbyte(s);
@@ -2636,6 +2999,10 @@ store_flush_position(XLogRecPtr remote_lsn)
{
FlushPosition *flushpos;
+ /* We only need to collect the LSN in main apply worker */
+ if (am_apply_bgworker())
+ return;
+
/* Need to do this in permanent context */
MemoryContextSwitchTo(ApplyContext);
@@ -2650,7 +3017,7 @@ store_flush_position(XLogRecPtr remote_lsn)
/* Update statistics of the worker. */
-static void
+void
UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply)
{
MyLogicalRepWorker->last_lsn = last_lsn;
@@ -2812,6 +3179,9 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
AcceptInvalidationMessages();
maybe_reread_subscription();
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any table synchronization changes. */
process_syncing_tables(last_received);
}
@@ -3113,7 +3483,7 @@ maybe_reread_subscription(void)
/*
* Callback from subscription syscache invalidation.
*/
-static void
+void
subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue)
{
MySubscriptionValid = false;
@@ -3709,7 +4079,7 @@ ApplyWorkerMain(Datum main_arg)
originid = replorigin_by_name(originname, true);
if (!OidIsValid(originid))
originid = replorigin_create(originname);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
origin_startpos = replorigin_session_get_progress(false);
CommitTransactionCommand();
@@ -3750,13 +4120,14 @@ ApplyWorkerMain(Datum main_arg)
server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
options.proto.logical.proto_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
LOGICALREP_PROTO_VERSION_NUM;
options.proto.logical.publication_names = MySubscription->publications;
options.proto.logical.binary = MySubscription->binary;
- options.proto.logical.streaming = MySubscription->stream;
+ options.proto.logical.streaming = (MySubscription->stream != SUBSTREAM_OFF);
options.proto.logical.twophase = false;
if (!am_tablesync_worker())
@@ -3914,7 +4285,8 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
XLogRecPtr myskiplsn = MySubscription->skiplsn;
bool started_tx = false;
- if (likely(XLogRecPtrIsInvalid(myskiplsn)))
+ if (likely(XLogRecPtrIsInvalid(myskiplsn)) ||
+ am_apply_bgworker())
return;
if (!IsTransactionState())
@@ -3986,7 +4358,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
}
/* Error callback to give more context info about the change being applied */
-static void
+void
apply_error_callback(void *arg)
{
ApplyErrorCallbackArg *errarg = &apply_error_callback_arg;
@@ -4014,23 +4386,47 @@ apply_error_callback(void *arg)
errarg->remote_xid,
LSN_FORMAT_ARGS(errarg->finish_lsn));
}
- else if (errarg->remote_attnum < 0)
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
else
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->rel->remoterel.attnames[errarg->remote_attnum],
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
+ {
+ if (errarg->remote_attnum < 0)
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ else
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ }
}
/* Set transaction information of apply error callback */
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 2cbca4a087..1aaca04982 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -1820,6 +1820,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
XLogRecPtr abort_lsn)
{
ReorderBufferTXN *toptxn;
+ bool write_abort_lsn = false;
+ PGOutputData *data = (PGOutputData *) ctx->output_plugin_private;
/*
* The abort should happen outside streaming block, even for streamed
@@ -1832,8 +1834,13 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
Assert(rbtxn_is_streamed(toptxn));
+ /* We only send abort_lsn and abort_time if the subscriber needs them. */
+ if (data->protocol_version >= LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM)
+ write_abort_lsn = true;
+
OutputPluginPrepareWrite(ctx, true);
- logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid);
+ logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn, abort_lsn,
+ write_abort_lsn);
OutputPluginWrite(ctx, true);
cleanup_rel_sync_cache(toptxn->xid, false);
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index 87c15b9c6f..ba781e6f08 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -388,6 +388,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT:
event_name = "HashGrowBucketsReinsert";
break;
+ case WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE:
+ event_name = "LogicalApplyWorkerStateChange";
+ break;
case WAIT_EVENT_LOGICAL_SYNC_DATA:
event_name = "LogicalSyncData";
break;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d43..4284bcbcd1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3220,6 +3220,18 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_apply_bgworkers_per_subscription",
+ PGC_SIGHUP,
+ REPLICATION_SUBSCRIBERS,
+ gettext_noop("Maximum number of apply background workers per subscription."),
+ NULL,
+ },
+ &max_apply_bgworkers_per_subscription,
+ 2, 0, MAX_BACKENDS,
+ NULL, NULL, NULL
+ },
+
{
{"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Sets the amount of time to wait before forcing "
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..ad18710af4 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,7 @@
#max_logical_replication_workers = 4 # taken from max_worker_processes
# (change requires restart)
#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers
+#max_apply_bgworkers_per_subscription = 2 # taken from max_logical_replication_workers
#------------------------------------------------------------------------------
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f317f0a681..24927641b9 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4449,7 +4449,7 @@ getSubscriptions(Archive *fout)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query, " s.substream,\n");
else
- appendPQExpBufferStr(query, " false AS substream,\n");
+ appendPQExpBufferStr(query, " 'f' AS substream,\n");
if (fout->remoteVersion >= 150000)
appendPQExpBufferStr(query,
@@ -4579,8 +4579,10 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo)
if (strcmp(subinfo->subbinary, "t") == 0)
appendPQExpBufferStr(query, ", binary = true");
- if (strcmp(subinfo->substream, "f") != 0)
+ if (strcmp(subinfo->substream, "t") == 0)
appendPQExpBufferStr(query, ", streaming = on");
+ else if (strcmp(subinfo->substream, "p") == 0)
+ appendPQExpBufferStr(query, ", streaming = parallel");
if (strcmp(subinfo->subtwophasestate, two_phase_disabled) != 0)
appendPQExpBufferStr(query, ", two_phase = on");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d1260f590c..d54540f5f5 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -68,7 +68,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subbinary; /* True if the subscription wants the
* publisher to send data in binary */
- bool substream; /* Stream in-progress transactions. */
+ char substream; /* Stream in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char subtwophasestate; /* Stream two-phase transactions */
@@ -109,7 +110,8 @@ typedef struct Subscription
bool enabled; /* Indicates if the subscription is enabled */
bool binary; /* Indicates if the subscription wants data in
* binary format */
- bool stream; /* Allow streaming in-progress transactions. */
+ char stream; /* Allow streaming in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char twophasestate; /* Allow streaming two-phase transactions */
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
@@ -120,6 +122,21 @@ typedef struct Subscription
List *publications; /* List of publication names to subscribe to */
} Subscription;
+/* Disallow streaming in-progress transactions */
+#define SUBSTREAM_OFF 'f'
+
+/*
+ * Streaming in-progress transactions are written to a temporary file and
+ * applied only after the transaction is committed on upstream.
+ */
+#define SUBSTREAM_ON 't'
+
+/*
+ * Streaming in-progress transactions are applied immediately via a background
+ * worker
+ */
+#define SUBSTREAM_PARALLEL 'p'
+
extern Subscription *GetSubscription(Oid subid, bool missing_ok);
extern void FreeSubscription(Subscription *sub);
extern void DisableSubscription(Oid subid);
diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h
index f1e2821e25..ac8ef94381 100644
--- a/src/include/replication/logicallauncher.h
+++ b/src/include/replication/logicallauncher.h
@@ -14,6 +14,7 @@
extern PGDLLIMPORT int max_logical_replication_workers;
extern PGDLLIMPORT int max_sync_workers_per_subscription;
+extern PGDLLIMPORT int max_apply_bgworkers_per_subscription;
extern void ApplyLauncherRegister(void);
extern void ApplyLauncherMain(Datum main_arg);
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index a771ab8ff3..0f74a3392b 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -32,12 +32,17 @@
*
* LOGICALREP_PROTO_TWOPHASE_VERSION_NUM is the minimum protocol version with
* support for two-phase commit decoding (at prepare time). Introduced in PG15.
+ *
+ * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version
+ * with support for streaming large transactions in apply background worker.
+ * Introduced in PG16.
*/
#define LOGICALREP_PROTO_MIN_VERSION_NUM 1
#define LOGICALREP_PROTO_VERSION_NUM 1
#define LOGICALREP_PROTO_STREAM_VERSION_NUM 2
#define LOGICALREP_PROTO_TWOPHASE_VERSION_NUM 3
-#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_TWOPHASE_VERSION_NUM
+#define LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM 4
+#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM
/*
* Logical message types
@@ -175,6 +180,17 @@ typedef struct LogicalRepRollbackPreparedTxnData
char gid[GIDSIZE];
} LogicalRepRollbackPreparedTxnData;
+/*
+ * Transaction protocol information for stream abort.
+ */
+typedef struct LogicalRepStreamAbortData
+{
+ TransactionId xid;
+ TransactionId subxid;
+ XLogRecPtr abort_lsn;
+ TimestampTz abort_time;
+} LogicalRepStreamAbortData;
+
extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn);
extern void logicalrep_read_begin(StringInfo in,
LogicalRepBeginData *begin_data);
@@ -246,9 +262,12 @@ extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn
extern TransactionId logicalrep_read_stream_commit(StringInfo out,
LogicalRepCommitData *commit_data);
extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid);
-extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid);
+ ReorderBufferTXN *txn,
+ XLogRecPtr abort_lsn,
+ bool write_abort_lsn);
+extern void logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool include_abort_lsn);
extern char *logicalrep_message_type(LogicalRepMsgType action);
#endif /* LOGICAL_PROTO_H */
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index cd1b6e8afc..6a1af7f13c 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -13,6 +13,7 @@
#define LOGICALWORKER_H
extern void ApplyWorkerMain(Datum main_arg);
+extern void ApplyBgworkerMain(Datum main_arg);
extern bool IsLogicalWorker(void);
diff --git a/src/include/replication/origin.h b/src/include/replication/origin.h
index 2d1b5e5c28..c7389b40a7 100644
--- a/src/include/replication/origin.h
+++ b/src/include/replication/origin.h
@@ -53,7 +53,7 @@ extern XLogRecPtr replorigin_get_progress(RepOriginId node, bool flush);
extern void replorigin_session_advance(XLogRecPtr remote_commit,
XLogRecPtr local_commit);
-extern void replorigin_session_setup(RepOriginId node);
+extern void replorigin_session_setup(RepOriginId node, bool must_acquire);
extern void replorigin_session_reset(void);
extern XLogRecPtr replorigin_session_get_progress(bool flush);
diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index d109d0baed..d2a80d79e5 100644
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -301,6 +301,7 @@ typedef struct ReorderBufferTXN
{
TimestampTz commit_time;
TimestampTz prepare_time;
+ TimestampTz abort_time;
} xact_time;
/*
@@ -647,9 +648,11 @@ extern void ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
extern void ReorderBufferAssignChild(ReorderBuffer *, TransactionId, TransactionId, XLogRecPtr commit_lsn);
extern void ReorderBufferCommitChild(ReorderBuffer *, TransactionId, TransactionId,
XLogRecPtr commit_lsn, XLogRecPtr end_lsn);
-extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferAbortOld(ReorderBuffer *, TransactionId xid);
-extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferInvalidate(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
extern void ReorderBufferSetBaseSnapshot(ReorderBuffer *, TransactionId, XLogRecPtr lsn, struct SnapshotData *snap);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..5be8f5755e 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -17,8 +17,11 @@
#include "access/xlogdefs.h"
#include "catalog/pg_subscription.h"
#include "datatype/timestamp.h"
+#include "replication/logicalrelation.h"
#include "storage/fileset.h"
#include "storage/lock.h"
+#include "storage/shm_mq.h"
+#include "storage/shm_toc.h"
#include "storage/spin.h"
@@ -60,6 +63,9 @@ typedef struct LogicalRepWorker
*/
FileSet *stream_fileset;
+ /* Indicates if this slot is used for an apply background worker. */
+ bool subworker;
+
/* Stats. */
XLogRecPtr last_lsn;
TimestampTz last_send_time;
@@ -68,8 +74,68 @@ typedef struct LogicalRepWorker
TimestampTz reply_time;
} LogicalRepWorker;
+/* Struct for saving and restoring apply errcontext information */
+typedef struct ApplyErrorCallbackArg
+{
+ LogicalRepMsgType command; /* 0 if invalid */
+ LogicalRepRelMapEntry *rel;
+
+ /* Remote node information */
+ int remote_attnum; /* -1 if invalid */
+ TransactionId remote_xid;
+ XLogRecPtr finish_lsn;
+ char *origin_name;
+} ApplyErrorCallbackArg;
+
+/*
+ * Status of apply background worker.
+ */
+typedef enum ApplyBgworkerStatus
+{
+ APPLY_BGWORKER_BUSY = 0, /* assigned to a transaction */
+ APPLY_BGWORKER_FINISHED, /* transaction is completed */
+ APPLY_BGWORKER_EXIT /* exit */
+} ApplyBgworkerStatus;
+
+/*
+ * Struct for sharing information between apply main and apply background
+ * workers.
+ */
+typedef struct ApplyBgworkerShared
+{
+ slock_t mutex;
+
+ /* Status of apply background worker. */
+ ApplyBgworkerStatus status;
+
+ /* server version of publisher. */
+ int server_version;
+
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
+
+/*
+ * Struct for maintaining an apply background worker.
+ */
+typedef struct ApplyBgworkerState
+{
+ shm_mq_handle *mq_handle;
+ dsm_segment *dsm_seg;
+ ApplyBgworkerShared volatile *pstate;
+} ApplyBgworkerState;
+
/* Main memory context for apply worker. Permanent during worker lifetime. */
extern PGDLLIMPORT MemoryContext ApplyContext;
+extern PGDLLIMPORT MemoryContext ApplyMessageContext;
+
+extern PGDLLIMPORT ApplyErrorCallbackArg apply_error_callback_arg;
+
+extern PGDLLIMPORT bool MySubscriptionValid;
+
+extern PGDLLIMPORT volatile ApplyBgworkerShared *MyParallelState;
+
+extern PGDLLIMPORT List *subxactlist;
/* libpqreceiver connection */
extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn;
@@ -79,18 +145,22 @@ extern PGDLLIMPORT Subscription *MySubscription;
extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker;
extern PGDLLIMPORT bool in_remote_transaction;
+extern PGDLLIMPORT bool in_streamed_transaction;
+extern PGDLLIMPORT TransactionId stream_xid;
extern void logicalrep_worker_attach(int slot);
extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
bool only_running);
extern List *logicalrep_workers_find(Oid subid, bool only_running);
-extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
- Oid userid, Oid relid);
+extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
+ Oid userid, Oid relid,
+ dsm_handle subworker_dsm);
extern void logicalrep_worker_stop(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
extern int logicalrep_sync_worker_count(Oid subid);
+extern int logicalrep_apply_bgworker_count(Oid subid);
extern void ReplicationOriginNameForTablesync(Oid suboid, Oid relid,
char *originname, int szorgname);
@@ -103,10 +173,38 @@ extern void process_syncing_tables(XLogRecPtr current_lsn);
extern void invalidate_syncing_table_states(Datum arg, int cacheid,
uint32 hashvalue);
+extern void UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time,
+ bool reply);
+
+extern void apply_dispatch(StringInfo s);
+
+/* Function for apply error callback */
+extern void apply_error_callback(void *arg);
+
+extern void subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue);
+
+/* Apply background worker setup and interactions */
+extern ApplyBgworkerState *apply_bgworker_start(TransactionId xid);
+extern ApplyBgworkerState *apply_bgworker_find(TransactionId xid);
+extern void apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status);
+extern void apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes,
+ const void *data);
+extern void apply_bgworker_free(ApplyBgworkerState *wstate);
+extern void apply_bgworker_check_status(void);
+extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
+extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+
static inline bool
am_tablesync_worker(void)
{
return OidIsValid(MyLogicalRepWorker->relid);
}
+static inline bool
+am_apply_bgworker(void)
+{
+ return MyLogicalRepWorker->subworker;
+}
+
#endif /* WORKER_INTERNAL_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b578e2ec75..c2d2a114d7 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -105,6 +105,7 @@ typedef enum
WAIT_EVENT_HASH_GROW_BUCKETS_ALLOCATE,
WAIT_EVENT_HASH_GROW_BUCKETS_ELECT,
WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE,
WAIT_EVENT_LOGICAL_SYNC_DATA,
WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE,
WAIT_EVENT_MQ_INTERNAL,
diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out
index 5db7146e06..919266ae06 100644
--- a/src/test/regress/expected/subscription.out
+++ b/src/test/regress/expected/subscription.out
@@ -197,7 +197,7 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
DROP SUBSCRIPTION regress_testsub;
-- fail - streaming must be boolean
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
-ERROR: streaming requires a Boolean value
+ERROR: streaming requires a Boolean value or "parallel"
-- now it works
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 34a76ceb60..4137dc77b4 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -116,6 +116,10 @@ Append
AppendPath
AppendRelInfo
AppendState
+ApplyBgworkerEntry
+ApplyBgworkerShared
+ApplyBgworkerState
+ApplyBgworkerStatus
ApplyErrorCallbackArg
ApplyExecutionData
ApplySubXactData
@@ -1485,6 +1489,7 @@ LogicalRepRelId
LogicalRepRelMapEntry
LogicalRepRelation
LogicalRepRollbackPreparedTxnData
+LogicalRepStreamAbortData
LogicalRepTupleData
LogicalRepTyp
LogicalRepWorker
--
2.23.0.windows.1
[application/octet-stream] v15-0002-Test-streaming-parallel-option-in-tap-test.patch (69.1K, ../../OS3PR01MB62755C6C9A75EB09F7218B589E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-v15-0002-Test-streaming-parallel-option-in-tap-test.patch)
download | inline diff:
From fcffbe96cfb5e38ff3befd46b12b6dfdda9ba843 Mon Sep 17 00:00:00 2001
From: "shiy.fnst" <[email protected]>
Date: Fri, 13 May 2022 14:50:30 +0800
Subject: [PATCH v15 2/4] Test streaming parallel option in tap test
Change all TAP tests using the SUBSCRIPTION "streaming" parameter, so they
now test both 'on' and 'parallel' values.
---
src/test/subscription/t/015_stream.pl | 199 ++++---
src/test/subscription/t/016_stream_subxact.pl | 119 +++--
src/test/subscription/t/017_stream_ddl.pl | 188 ++++---
.../t/018_stream_subxact_abort.pl | 195 ++++---
.../t/019_stream_subxact_ddl_abort.pl | 110 +++-
.../subscription/t/022_twophase_cascade.pl | 363 +++++++------
.../subscription/t/023_twophase_stream.pl | 498 ++++++++++--------
7 files changed, 1035 insertions(+), 637 deletions(-)
diff --git a/src/test/subscription/t/015_stream.pl b/src/test/subscription/t/015_stream.pl
index 6561b189de..0bdd234935 100644
--- a/src/test/subscription/t/015_stream.pl
+++ b/src/test/subscription/t/015_stream.pl
@@ -8,6 +8,116 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Interleave a pair of transactions, each exceeding the 64kB limit.
+ my $in = '';
+ my $out = '';
+
+ my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
+
+ my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
+ on_error_stop => 0);
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ $in .= q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ };
+ $h->pump_nb;
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
+ DELETE FROM test_tab WHERE a > 5000;
+ COMMIT;
+ });
+
+ $in .= q{
+ COMMIT;
+ \q
+ };
+ $h->finish; # errors make the next test fail, so ignore them here
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'check extra columns contain local defaults');
+
+ # Test the streaming in binary mode
+ $node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain local defaults');
+
+ # Change the local values of the extra columns on the subscriber,
+ # update publisher, and check that subscriber retains the expected
+ # values. This is to ensure that non-streaming transactions behave
+ # properly after a streaming transaction.
+ $node_subscriber->safe_psql('postgres',
+ "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
+ );
+ $node_publisher->safe_psql('postgres',
+ "UPDATE test_tab SET b = md5(a::text)");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
+ );
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain locally changed data');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +147,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,82 +168,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Interleave a pair of transactions, each exceeding the 64kB limit.
-my $in = '';
-my $out = '';
-
-my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
-
-my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
- on_error_stop => 0);
-
-$in .= q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-};
-$h->pump_nb;
-
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
-DELETE FROM test_tab WHERE a > 5000;
-COMMIT;
-});
-
-$in .= q{
-COMMIT;
-\q
-};
-$h->finish; # errors make the next test fail, so ignore them here
-
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334), 'check extra columns contain local defaults');
-
-# Test the streaming in binary mode
-$node_subscriber->safe_psql('postgres',
- "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(6667|6667|6667), 'check extra columns contain local defaults');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+);
-# Change the local values of the extra columns on the subscriber,
-# update publisher, and check that subscriber retains the expected
-# values. This is to ensure that non-streaming transactions behave
-# properly after a streaming transaction.
$node_subscriber->safe_psql('postgres',
- "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
-);
-$node_publisher->safe_psql('postgres',
- "UPDATE test_tab SET b = md5(a::text)");
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel, binary = off)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
-);
-is($result, qq(6667|6667|6667),
- 'check extra columns contain locally changed data');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/016_stream_subxact.pl b/src/test/subscription/t/016_stream_subxact.pl
index f27f1694f2..45429dddba 100644
--- a/src/test/subscription/t/016_stream_subxact.pl
+++ b/src/test/subscription/t/016_stream_subxact.pl
@@ -8,6 +8,72 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(1667|1667|1667),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +103,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,41 +124,26 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Insert, update and delete enough rows to exceed 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(1667|1667|1667),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/017_stream_ddl.pl b/src/test/subscription/t/017_stream_ddl.pl
index 0bce63b716..52dfef4780 100644
--- a/src/test/subscription/t/017_stream_ddl.pl
+++ b/src/test/subscription/t/017_stream_ddl.pl
@@ -8,6 +8,111 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (3, md5(3::text));
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (4, md5(4::text), -4);
+ COMMIT;
+ });
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
+ is($result, qq(2002|1999|1002|1),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # A large (streamed) transaction with DDL and DML. One of the DDL is performed
+ # after DML to ensure that we invalidate the schema sent for test_tab so that
+ # the next transaction has to send the schema again.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
+ ALTER TABLE test_tab ADD COLUMN f INT;
+ COMMIT;
+ });
+
+ # A small transaction that won't get streamed. This is just to ensure that we
+ # send the schema again to reflect the last column added in the previous test.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab"
+ );
+ is($result, qq(5005|5002|4005|3004|5),
+ 'check data was copied to subscriber for both streaming and non-streaming transactions'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c, DROP COLUMN d, DROP COLUMN e, DROP COLUMN f;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +142,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,76 +163,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|0|0), 'check initial data was copied to subscriber');
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (3, md5(3::text));
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (4, md5(4::text), -4);
-COMMIT;
-});
-
-# large (streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
-COMMIT;
-});
-
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
-is($result, qq(2002|1999|1002|1),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# A large (streamed) transaction with DDL and DML. One of the DDL is performed
-# after DML to ensure that we invalidate the schema sent for test_tab so that
-# the next transaction has to send the schema again.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
-ALTER TABLE test_tab ADD COLUMN f INT;
-COMMIT;
-});
-
-# A small transaction that won't get streamed. This is just to ensure that we
-# send the schema again to reflect the last column added in the previous test.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
-COMMIT;
-});
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab");
-is($result, qq(5005|5002|4005|3004|5),
- 'check data was copied to subscriber for both streaming and non-streaming transactions'
-);
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/018_stream_subxact_abort.pl b/src/test/subscription/t/018_stream_subxact_abort.pl
index 7155442e76..68f0e4b0d1 100644
--- a/src/test/subscription/t/018_stream_subxact_abort.pl
+++ b/src/test/subscription/t/018_stream_subxact_abort.pl
@@ -8,6 +8,113 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
+ ROLLBACK TO s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
+ SAVEPOINT s5;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2000|0),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # large (streamed) transaction with subscriber receiving out of order
+ # subtransaction ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
+ RELEASE s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
+ ROLLBACK TO s1;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0),
+ 'check rollback to savepoint was reflected on subscriber');
+
+ # large (streamed) transaction with subscriber receiving rollback
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
+ ROLLBACK;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -36,6 +143,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -53,81 +164,25 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
-ROLLBACK TO s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
-SAVEPOINT s5;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2000|0),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# large (streamed) transaction with subscriber receiving out of order
-# subtransaction ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
-RELEASE s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
-ROLLBACK TO s1;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0),
- 'check rollback to savepoint was reflected on subscriber');
-
-# large (streamed) transaction with subscriber receiving rollback
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
-ROLLBACK;
-});
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
index dbd0fca4d1..b276063721 100644
--- a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
+++ b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
@@ -9,6 +9,69 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
+ ALTER TABLE test_tab DROP COLUMN c;
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(1000|500),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +100,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,35 +121,26 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
-ALTER TABLE test_tab DROP COLUMN c;
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(1000|500),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 7a797f37ba..0a4152d3be 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -11,6 +11,208 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming" parameter
+# so the same code can be run both for the streaming=on and streaming=parallel
+# cases.
+sub test_streaming
+{
+ my ($node_A, $node_B, $node_C, $appname_B, $appname_C, $streaming_mode) =
+ @_;
+
+ my $oldpid_B = $node_A->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';");
+ my $oldpid_C = $node_B->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';");
+
+ # Setup logical replication streaming mode
+
+ $node_B->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_B
+ SET (streaming = $streaming_mode);");
+ $node_C->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_C
+ SET (streaming = $streaming_mode)");
+
+ # Wait for subscribers to finish initialization
+
+ $node_A->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_B FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+ $node_B->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_C FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber(s) after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" optparameterion is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_B->reload;
+
+ $node_C->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_C->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ # Then 2PC PREPARE
+ $node_A->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_B->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_B->reload;
+
+ $node_C->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_C->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_C->reload;
+ }
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is prepared on subscriber(s)
+ my $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check that transaction was committed on subscriber(s)
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
+ );
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
+ );
+
+ # check the transaction state is ended on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber C');
+
+ ###############################
+ # Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
+ # 0. Cleanup from previous test leaving only 2 rows.
+ # 1. Insert one more row.
+ # 2. Record a SAVEPOINT.
+ # 3. Data is streamed using 2PC.
+ # 4. Do rollback to SAVEPOINT prior to the streamed inserts.
+ # 5. Then COMMIT PREPARED.
+ #
+ # Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (delete will be replicated)
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+
+ # 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
+ $node_A->safe_psql(
+ 'postgres', "
+ BEGIN;
+ INSERT INTO test_tab VALUES (9999, 'foobar');
+ SAVEPOINT sp_inner;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ ROLLBACK TO SAVEPOINT sp_inner;
+ PREPARE TRANSACTION 'outer';
+ ");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state prepared on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is ended on subscriber
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber C');
+
+ # check inserts are visible at subscriber(s).
+ # All the streamed data (prior to the SAVEPOINT) should be rolled back.
+ # (9999, 'foobar') should be committed.
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber B');
+ $result =
+ $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber B');
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber C');
+ $result =
+ $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber C');
+
+ # Cleanup the test data
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+}
+
###############################
# Setup a cascade of pub/sub nodes.
# node_A -> node_B -> node_C
@@ -260,160 +462,15 @@ is($result, qq(21), 'Rows committed are present on subscriber C');
# 2PC + STREAMING TESTS
# ---------------------
-my $oldpid_B = $node_A->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';");
-my $oldpid_C = $node_B->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';");
-
-# Setup logical replication (streaming = on)
-
-$node_B->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_B
- SET (streaming = on);");
-$node_C->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_C
- SET (streaming = on)");
-
-# Wait for subscribers to finish initialization
-
-$node_A->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_B FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-$node_B->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_C FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber(s) after the commit.
-###############################
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-# Then 2PC PREPARE
-$node_A->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
+################################
+# Test using streaming mode 'on'
+################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'on');
-# check the transaction state is prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check that transaction was committed on subscriber(s)
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
-);
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
-);
-
-# check the transaction state is ended on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber C');
-
-###############################
-# Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
-# 0. Cleanup from previous test leaving only 2 rows.
-# 1. Insert one more row.
-# 2. Record a SAVEPOINT.
-# 3. Data is streamed using 2PC.
-# 4. Do rollback to SAVEPOINT prior to the streamed inserts.
-# 5. Then COMMIT PREPARED.
-#
-# Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
-###############################
-
-# First, delete the data except for 2 rows (delete will be replicated)
-$node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
-$node_A->safe_psql(
- 'postgres', "
- BEGIN;
- INSERT INTO test_tab VALUES (9999, 'foobar');
- SAVEPOINT sp_inner;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- ROLLBACK TO SAVEPOINT sp_inner;
- PREPARE TRANSACTION 'outer';
- ");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state is ended on subscriber
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber C');
-
-# check inserts are visible at subscriber(s).
-# All the streamed data (prior to the SAVEPOINT) should be rolled back.
-# (9999, 'foobar') should be committed.
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber B');
-$result = $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber B');
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber C');
-$result = $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber C');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'parallel');
###############################
# check all the cleanup
diff --git a/src/test/subscription/t/023_twophase_stream.pl b/src/test/subscription/t/023_twophase_stream.pl
index d8475d25a4..b89414ab74 100644
--- a/src/test/subscription/t/023_twophase_stream.pl
+++ b/src/test/subscription/t/023_twophase_stream.pl
@@ -8,6 +8,266 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # check that 2PC gets replicated to subscriber
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ my $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ ###############################
+ # Test 2PC PREPARE / ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. Do rollback prepared.
+ #
+ # Expect data rolls back leaving only the original 2 rows.
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(2|2|2),
+ 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
+ # 1. insert, update and delete enough rows to exceed the 64kB limit.
+ # 2. Then server crashes before the 2PC transaction is committed.
+ # 3. After servers are restarted the pending transaction is committed.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ # Note: both publisher and subscriber do crash/restart.
+ ###############################
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_subscriber->stop('immediate');
+ $node_publisher->stop('immediate');
+
+ $node_publisher->start;
+ $node_subscriber->start;
+
+ # commit post the restart
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+ $node_publisher->wait_for_catchup($appname);
+
+ # check inserts are visible
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ ###############################
+ # Do INSERT after the PREPARE but before ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a ROLLBACK PREPARED.
+ #
+ # Expect the 2PC data rolls back leaving only 3 rows on the subscriber
+ # (the original 2 + inserted 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber,
+ # but the extra INSERT outside of the 2PC still was replicated
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3|3|3),
+ 'check the outside insert was copied to subscriber');
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Do INSERT after the PREPARE but before COMMIT PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a COMMIT PREPARED.
+ #
+ # Expect 2PC data + the extra row are on the subscriber
+ # (the 3334 + inserted 1 = 3335).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3335|3335|3335),
+ 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+ $node_publisher->wait_for_catchup($appname);
+}
+
###############################
# Setup
###############################
@@ -48,6 +308,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql(
'postgres', "
CREATE SUBSCRIPTION tap_sub
@@ -70,236 +334,30 @@ my $twophase_query =
$node_subscriber->poll_query_until('postgres', $twophase_query)
or die "Timed out while waiting for subscriber to enable twophase";
-###############################
# Check initial data was copied to subscriber
-###############################
my $result = $node_subscriber->safe_psql('postgres',
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber side after the commit.
-###############################
-
-# check that 2PC gets replicated to subscriber
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
-);
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
-
-###############################
-# Test 2PC PREPARE / ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. Do rollback prepared.
-#
-# Expect data rolls back leaving only the original 2 rows.
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(2|2|2),
- 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
-# 1. insert, update and delete enough rows to exceed the 64kB limit.
-# 2. Then server crashes before the 2PC transaction is committed.
-# 3. After servers are restarted the pending transaction is committed.
-#
-# Expect all data is replicated on subscriber side after the commit.
-# Note: both publisher and subscriber do crash/restart.
-###############################
-
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_subscriber->stop('immediate');
-$node_publisher->stop('immediate');
-
-$node_publisher->start;
-$node_subscriber->start;
-
-# commit post the restart
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-# check inserts are visible
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-###############################
-# Do INSERT after the PREPARE but before ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a ROLLBACK PREPARED.
-#
-# Expect the 2PC data rolls back leaving only 3 rows on the subscriber
-# (the original 2 + inserted 1).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber,
-# but the extra INSERT outside of the 2PC still was replicated
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3|3|3), 'check the outside insert was copied to subscriber');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Do INSERT after the PREPARE but before COMMIT PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a COMMIT PREPARED.
-#
-# Expect 2PC data + the extra row are on the subscriber
-# (the 3334 + inserted 1 = 3335).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3335|3335|3335),
- 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
-);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
###############################
# check all the cleanup
--
2.23.0.windows.1
[application/octet-stream] v15-0003-Add-some-checks-before-using-apply-background-wo.patch (36.6K, ../../OS3PR01MB62755C6C9A75EB09F7218B589E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-v15-0003-Add-some-checks-before-using-apply-background-wo.patch)
download | inline diff:
From 062b9c2151da30b04e3e9f26f6adde82a3462f3c Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 11:23:52 +0800
Subject: [PATCH v15 3/4] Add some checks before using apply background worker
to apply changes.
streaming=parallel mode has two requirements:
1) The unique column in the relation on the subscriber-side should also be the
unique column on the publisher-side;
2) There cannot be any non-immutable functions in the subscriber-side
replicated table. Look for functions in the following places:
* a. Trigger functions
* b. Column default value expressions and domain constraints
* c. Constraint expressions
* d. Foreign keys
---
doc/src/sgml/ref/create_subscription.sgml | 4 +
.../replication/logical/applybgworker.c | 42 ++
src/backend/replication/logical/proto.c | 66 ++-
src/backend/replication/logical/relation.c | 199 +++++++++
src/backend/replication/logical/tablesync.c | 1 +
src/backend/replication/logical/worker.c | 23 +-
src/backend/utils/cache/typcache.c | 17 +
src/include/replication/logicalproto.h | 1 +
src/include/replication/logicalrelation.h | 15 +
src/include/replication/worker_internal.h | 1 +
src/include/utils/typcache.h | 2 +
.../subscription/t/022_twophase_cascade.pl | 6 +
.../subscription/t/032_streaming_apply.pl | 391 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
14 files changed, 759 insertions(+), 10 deletions(-)
create mode 100644 src/test/subscription/t/032_streaming_apply.pl
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 71dd4aca81..270e3d382e 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -240,6 +240,10 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
transaction is committed. Note that if an error happens when
applying changes in a background worker, the finish LSN of the
remote transaction might not be reported in the server log.
+ Parallel mode has two requirements: 1) the unique column in the
+ relation on the subscriber-side should also be the unique column on
+ the publisher-side; 2) there cannot be any non-immutable functions
+ in the subscriber-side replicated table.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index dfa49be98c..92b4b4e4ed 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -773,3 +773,45 @@ apply_bgworker_subxact_info_add(TransactionId current_xid)
MemoryContextSwitchTo(oldctx);
}
}
+
+/*
+ * Check if changes on this relation can be applied by an apply background
+ * worker.
+ *
+ * Although the commit order is maintained only allowing one process to commit
+ * at a time, the access order to the relation has changed. This could cause
+ * unexpected problems if the unique column on the replicated table is
+ * inconsistent with the publisher-side or contains non-immutable functions
+ * when applying transactions in the apply background worker.
+ */
+void
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
+{
+ /* Skip check if not an apply background worker. */
+ if (!am_apply_bgworker())
+ return;
+
+ /*
+ * Partition table checks are done later in function
+ * apply_handle_tuple_routing.
+ */
+ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ return;
+
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel == PARALLEL_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot replicate target relation \"%s.%s\" in parallel "
+ "mode", rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change the streaming option to 'on' instead of 'parallel'.")));
+}
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index affd08cfa4..12a9799f5c 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -23,7 +23,8 @@
/*
* Protocol message flags.
*/
-#define LOGICALREP_IS_REPLICA_IDENTITY 1
+#define ATTR_IS_REPLICA_IDENTITY (1 << 0)
+#define ATTR_IS_UNIQUE (1 << 1)
#define MESSAGE_TRANSACTIONAL (1<<0)
#define TRUNCATE_CASCADE (1<<0)
@@ -933,11 +934,55 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
TupleDesc desc;
int i;
uint16 nliveatts = 0;
- Bitmapset *idattrs = NULL;
+ Bitmapset *idattrs = NULL,
+ *attunique = NULL;
bool replidentfull;
desc = RelationGetDescr(rel);
+ if (rel->rd_rel->relhasindex)
+ {
+ List *indexoidlist = RelationGetIndexList(rel);
+ ListCell *indexoidscan;
+
+ foreach(indexoidscan, indexoidlist)
+ {
+ Oid indexoid = lfirst_oid(indexoidscan);
+ Relation indexRel;
+
+ /* Look up the description for index */
+ indexRel = RelationIdGetRelation(indexoid);
+
+ if (!RelationIsValid(indexRel))
+ elog(ERROR, "could not open relation with OID %u", indexoid);
+
+ if (indexRel->rd_index->indisunique)
+ {
+ int i;
+
+ /* Add referenced attributes to idindexattrs */
+ for (i = 0; i < indexRel->rd_index->indnatts; i++)
+ {
+ int attrnum = indexRel->rd_index->indkey.values[i];
+
+ /*
+ * We don't include non-key columns into idindexattrs
+ * bitmaps. See RelationGetIndexAttrBitmap.
+ */
+ if (attrnum != 0)
+ {
+ if (i < indexRel->rd_index->indnkeyatts &&
+ !bms_is_member(attrnum - FirstLowInvalidHeapAttributeNumber, attunique))
+ attunique = bms_add_member(attunique,
+ attrnum - FirstLowInvalidHeapAttributeNumber);
+ }
+ }
+ }
+ RelationClose(indexRel);
+ }
+ list_free(indexoidlist);
+ }
+
/* send number of live attributes */
for (i = 0; i < desc->natts; i++)
{
@@ -974,7 +1019,11 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (replidentfull ||
bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
idattrs))
- flags |= LOGICALREP_IS_REPLICA_IDENTITY;
+ flags |= ATTR_IS_REPLICA_IDENTITY;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ attunique))
+ flags |= ATTR_IS_UNIQUE;
pq_sendbyte(out, flags);
@@ -1001,7 +1050,8 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
int natts;
char **attnames;
Oid *atttyps;
- Bitmapset *attkeys = NULL;
+ Bitmapset *attkeys = NULL,
+ *attunique = NULL;
natts = pq_getmsgint(in, 2);
attnames = palloc(natts * sizeof(char *));
@@ -1012,11 +1062,14 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
{
uint8 flags;
- /* Check for replica identity column */
+ /* Check for replica identity and unique column */
flags = pq_getmsgbyte(in);
- if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
+ if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
+ if (flags & ATTR_IS_UNIQUE)
+ attunique = bms_add_member(attunique, i);
+
/* attribute name */
attnames[i] = pstrdup(pq_getmsgstring(in));
@@ -1030,6 +1083,7 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
rel->attnames = attnames;
rel->atttyps = atttyps;
rel->attkeys = attkeys;
+ rel->attunique = attunique;
rel->natts = natts;
}
diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c
index e989047681..072de1557c 100644
--- a/src/backend/replication/logical/relation.c
+++ b/src/backend/replication/logical/relation.c
@@ -19,12 +19,19 @@
#include "access/table.h"
#include "catalog/namespace.h"
+#include "catalog/pg_proc.h"
#include "catalog/pg_subscription_rel.h"
+#include "commands/trigger.h"
#include "executor/executor.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "replication/logicalrelation.h"
#include "replication/worker_internal.h"
+#include "rewrite/rewriteHandler.h"
#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
static MemoryContext LogicalRepRelMapContext = NULL;
@@ -91,6 +98,26 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
}
}
+/*
+ * Relcache invalidation callback to reset parallel flag.
+ */
+static void
+logicalrep_relmap_reset_parallel_cb(Datum arg, int cacheid, uint32 hashvalue)
+{
+ HASH_SEQ_STATUS hash_seq;
+ LogicalRepRelMapEntry *entry;
+
+ if (LogicalRepRelMap == NULL)
+ return;
+
+ hash_seq_init(&hash_seq, LogicalRepRelMap);
+ while ((entry = hash_seq_search(&hash_seq)) != NULL)
+ {
+ entry->parallel = PARALLEL_UNKNOWN;
+ entry->localrelvalid = false;
+ }
+}
+
/*
* Initialize the relation map cache.
*/
@@ -116,6 +143,9 @@ logicalrep_relmap_init(void)
/* Watch for invalidation events. */
CacheRegisterRelcacheCallback(logicalrep_relmap_invalidate_cb,
(Datum) 0);
+ CacheRegisterSyscacheCallback(PROCOID,
+ logicalrep_relmap_reset_parallel_cb,
+ (Datum) 0);
}
/*
@@ -142,6 +172,7 @@ logicalrep_relmap_free_entry(LogicalRepRelMapEntry *entry)
pfree(remoterel->atttyps);
}
bms_free(remoterel->attkeys);
+ bms_free(remoterel->attunique);
if (entry->attrmap)
free_attrmap(entry->attrmap);
@@ -190,6 +221,7 @@ logicalrep_relmap_update(LogicalRepRelation *remoterel)
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
MemoryContextSwitchTo(oldctx);
}
@@ -310,6 +342,166 @@ logicalrep_rel_mark_updatable(LogicalRepRelMapEntry *entry)
}
}
+/*
+ * Check if changes on one relation can be applied by an apply background
+ * worker and assign the 'parallel' flag.
+ *
+ * There are two requirements for applying changes in an apply background
+ * worker: 1) The unique column in the relation on the subscriber-side should
+ * also be the unique column on the publisher-side; 2) There cannot be any
+ * non-immutable functions in the subscriber-side.
+ *
+ * We just mark the relation entry as 'PARALLEL_RESTRICTED' here if changes on
+ * one relation can not be applied by an apply background worker and leave it
+ * to apply_bgworker_relation_check() to throw the actual error if needed.
+ */
+static void
+logicalrep_rel_mark_parallel(LogicalRepRelMapEntry *entry)
+{
+ Bitmapset *ukey;
+ int i;
+ TupleDesc tupdesc;
+ int attnum;
+ List *fkeys = NIL;
+
+ /* Fast path if we marked 'parallel' flag. */
+ if (entry->parallel != PARALLEL_UNKNOWN)
+ return;
+
+ /* Initialize the flag. */
+ entry->parallel = PARALLEL_SAFE;
+
+ /*
+ * First, we check if the unique column in the relation on the
+ * subscriber-side is also the unique column on the publisher-side.
+ */
+ ukey = RelationGetIndexAttrBitmap(entry->localrel,
+ INDEX_ATTR_BITMAP_KEY);
+
+ if (ukey)
+ {
+ i = -1;
+ while ((i = bms_next_member(ukey, i)) >= 0)
+ {
+ attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
+
+ if (entry->attrmap->attnums[attnum] < 0 ||
+ !bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
+ {
+ entry->parallel = PARALLEL_RESTRICTED;
+ return;
+ }
+ }
+ }
+
+ /*
+ * Then, We check if there is any non-immutable function in the local
+ * table. Look for functions in the following places:
+ * a. trigger functions;
+ * b. Column default value expressions and domain constraints;
+ * c. Constraint expressions;
+ * d. Foreign keys.
+ */
+ /* Check the trigger functions. */
+ if (entry->localrel->trigdesc != NULL)
+ {
+ for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
+ {
+ Trigger *trig = entry->localrel->trigdesc->triggers + i;
+
+ if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
+ trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
+ continue;
+
+ if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
+ {
+ entry->parallel = PARALLEL_RESTRICTED;
+ return;
+ }
+ }
+ }
+
+ /* Check the columns. */
+ tupdesc = RelationGetDescr(entry->localrel);
+ for (attnum = 0; attnum < tupdesc->natts; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
+
+ /* We don't need info for dropped or generated attributes */
+ if (att->attisdropped || att->attgenerated)
+ continue;
+
+ /*
+ * We don't need to check columns that only exist on the
+ * subscriber
+ */
+ if (entry->attrmap->attnums[attnum] < 0)
+ continue;
+
+ if (att->atthasdef)
+ {
+ Node *defaultexpr;
+
+ defaultexpr = build_column_default(entry->localrel, attnum + 1);
+ if (contain_mutable_functions(defaultexpr))
+ {
+ entry->parallel = PARALLEL_RESTRICTED;
+ return;
+ }
+ }
+
+ /*
+ * If the column is of a DOMAIN type, determine whether
+ * that domain has any CHECK expressions that are not
+ * immutable.
+ */
+ if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
+ {
+ List *domain_constraints;
+ ListCell *lc;
+
+ domain_constraints = GetDomainConstraints(att->atttypid);
+
+ foreach(lc, domain_constraints)
+ {
+ DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
+
+ if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
+ {
+ entry->parallel = PARALLEL_RESTRICTED;
+ return;
+ }
+ }
+ }
+ }
+
+ /* Check the constraints. */
+ if (tupdesc->constr)
+ {
+ ConstrCheck *check = tupdesc->constr->check;
+
+ /*
+ * Determine if there are any CHECK constraints which
+ * contains non-immutable function.
+ */
+ for (i = 0; i < tupdesc->constr->num_check; i++)
+ {
+ Expr *check_expr = stringToNode(check[i].ccbin);
+
+ if (contain_mutable_functions((Node *) check_expr))
+ {
+ entry->parallel = PARALLEL_RESTRICTED;
+ return;
+ }
+ }
+ }
+
+ /* Check the foreign keys. */
+ fkeys = RelationGetFKeyList(entry->localrel);
+ if (fkeys)
+ entry->parallel = PARALLEL_RESTRICTED;
+}
+
/*
* Open the local relation associated with the remote one.
*
@@ -438,6 +630,9 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
*/
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel(entry);
+
entry->localrelvalid = true;
}
@@ -653,6 +848,7 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
}
entry->localrel = partrel;
@@ -696,6 +892,9 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
/* Set if the table's replica identity is enough to apply update/delete. */
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel(entry);
+
entry->localrelvalid = true;
/* state and statelsn are left set to 0. */
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8ffba7e2e5..3cdbf8b457 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -884,6 +884,7 @@ fetch_remote_table_info(char *nspname, char *relname,
lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
lrel->attkeys = NULL;
+ lrel->attunique = NULL;
/*
* Store the columns as a list of names. Ignore those that are not
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 9da11dd41c..81e0005d12 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -1388,6 +1388,14 @@ apply_handle_stream_stop(StringInfo s)
{
char action = LOGICAL_REP_MSG_STREAM_STOP;
+ /*
+ * Unlike stream_commit, we don't need to wait here for stream_stop to
+ * finish. Allowing the other transaction to be applied before
+ * stream_stop is finished can lead to failures if the unique
+ * index/constraint is different between publisher and subscriber. But
+ * for such cases, we don't allow streamed transactions to be applied
+ * in parallel. See apply_bgworker_relation_check.
+ */
apply_bgworker_send_data(stream_apply_worker, 1, &action);
elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
@@ -2040,6 +2048,8 @@ apply_handle_insert(StringInfo s)
/* Set relation for error callback */
apply_error_callback_arg.rel = rel;
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2183,6 +2193,8 @@ apply_handle_update(StringInfo s)
/* Check if we can do the update. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2351,6 +2363,8 @@ apply_handle_delete(StringInfo s)
/* Check if we can do the delete. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2536,13 +2550,14 @@ apply_handle_tuple_routing(ApplyExecutionData *edata,
}
MemoryContextSwitchTo(oldctx);
+ part_entry = logicalrep_partition_open(relmapentry, partrel,
+ attrmap);
+
/* Check if we can do the update or delete on the leaf partition. */
if (operation == CMD_UPDATE || operation == CMD_DELETE)
- {
- part_entry = logicalrep_partition_open(relmapentry, partrel,
- attrmap);
check_relation_updatable(part_entry);
- }
+
+ apply_bgworker_relation_check(part_entry);
switch (operation)
{
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 808f9ebd0d..b248899d82 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -2540,6 +2540,23 @@ compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2)
return 0;
}
+/*
+ * GetDomainConstraints --- get DomainConstraintState list of specified domain type
+ */
+List *
+GetDomainConstraints(Oid type_id)
+{
+ TypeCacheEntry *typentry;
+ List *constraints = NIL;
+
+ typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
+
+ if(typentry->domainData != NULL)
+ constraints = typentry->domainData->constraints;
+
+ return constraints;
+}
+
/*
* Load (or re-load) the enumData member of the typcache entry.
*/
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index 0f74a3392b..130c4eb82f 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -113,6 +113,7 @@ typedef struct LogicalRepRelation
char replident; /* replica identity */
char relkind; /* remote relation kind */
Bitmapset *attkeys; /* Bitmap of key columns */
+ Bitmapset *attunique; /* Bitmap of unique columns */
} LogicalRepRelation;
/* Type mapping info */
diff --git a/src/include/replication/logicalrelation.h b/src/include/replication/logicalrelation.h
index 78cd7e77f5..015545cd8d 100644
--- a/src/include/replication/logicalrelation.h
+++ b/src/include/replication/logicalrelation.h
@@ -15,6 +15,19 @@
#include "access/attmap.h"
#include "replication/logicalproto.h"
+/*
+ * States to determine if changes on one relation can be applied by an apply
+ * background worker.
+ */
+typedef enum RelParallel
+{
+ PARALLEL_UNKNOWN = 0, /* unknown */
+ PARALLEL_SAFE, /* Can apply changes in an apply background
+ worker */
+ PARALLEL_RESTRICTED /* Can not apply changes in an apply background
+ worker */
+} RelParallel;
+
typedef struct LogicalRepRelMapEntry
{
LogicalRepRelation remoterel; /* key is remoterel.remoteid */
@@ -31,6 +44,8 @@ typedef struct LogicalRepRelMapEntry
Relation localrel; /* relcache entry (NULL when closed) */
AttrMap *attrmap; /* map of local attributes to remote ones */
bool updatable; /* Can apply updates/deletes? */
+ RelParallel parallel; /* Can apply changes in an apply
+ background worker? */
/* Sync state. */
char state;
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 5be8f5755e..b28f4ab977 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -194,6 +194,7 @@ extern void apply_bgworker_free(ApplyBgworkerState *wstate);
extern void apply_bgworker_check_status(void);
extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+extern void apply_bgworker_relation_check(LogicalRepRelMapEntry *rel);
static inline bool
am_tablesync_worker(void)
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index 431ad7f1b3..ed7c2e7f48 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -199,6 +199,8 @@ extern uint64 assign_record_type_identifier(Oid type_id, int32 typmod);
extern int compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2);
+extern List *GetDomainConstraints(Oid type_id);
+
extern size_t SharedRecordTypmodRegistryEstimate(void);
extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 0a4152d3be..30a01f7305 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -39,6 +39,12 @@ sub test_streaming
ALTER SUBSCRIPTION tap_sub_C
SET (streaming = $streaming_mode)");
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_C->safe_psql(
+ 'postgres', "ALTER TABLE test_tab ALTER c DROP DEFAULT");
+ }
+
# Wait for subscribers to finish initialization
$node_A->poll_query_until(
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
new file mode 100644
index 0000000000..eca4328676
--- /dev/null
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -0,0 +1,391 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test the restrictions of streaming mode "parallel" in logical replication
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $offset = 0;
+
+# Create publisher node
+my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->append_conf('postgresql.conf',
+ 'logical_decoding_work_mem = 64kB');
+$node_publisher->start;
+
+# Create subscriber node
+my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber');
+$node_subscriber->init;
+$node_subscriber->start;
+
+# Setup structure on publisher
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar)");
+
+# Setup structure on subscriber
+# We need to test normal table and partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar) PARTITION BY RANGE(a)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partition (LIKE test_tab_partitioned)");
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partitioned ATTACH PARTITION test_tab_partition DEFAULT"
+);
+
+# Setup logical replication
+my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub FOR TABLE test_tab");
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub_partitioned FOR TABLE test_tab_partitioned");
+
+my $appname = 'tap_sub';
+$node_subscriber->safe_psql(
+ 'postgres', "
+ CREATE SUBSCRIPTION tap_sub
+ CONNECTION '$publisher_connstr application_name=$appname'
+ PUBLICATION tap_pub, tap_pub_partitioned
+ WITH (streaming = parallel, copy_data = false)");
+
+$node_publisher->wait_for_catchup($appname);
+
+# It is not allowed that the unique index on the publisher and the subscriber
+# is different. Check the error reported by background worker in this case.
+# First we check the unique index on normal table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
+
+# Check that a background worker starts if "streaming" option is specified as
+# "parallel". We have to look for the DEBUG1 log messages about that, so
+# temporarily bump up the log verbosity.
+$node_subscriber->append_conf('postgresql.conf', "log_min_messages = debug1");
+$node_subscriber->reload;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+$node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+$node_subscriber->reload;
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+my $result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Then we check the unique index on partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_partition_idx ON test_tab_partition (b)");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Triggers which execute non-immutable function are not allowed on the
+# subscriber side. Check the error reported by background worker in this case.
+# First we check the trigger function on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE FUNCTION trigger_func() RETURNS TRIGGER AS \$\$
+ BEGIN
+ RETURN NULL;
+ END
+\$\$ language plpgsql;
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# Then we check the trigger function on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab_partition
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab_partition ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# It is not allowed that column default value expression contains a
+# non-immutable function on the subscriber side. Check the error reported by
+# background worker in this case.
+# First we check the column default value expression on normal table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# Then we check the column default value expression on partition table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# It is not allowed that domain constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case.
+# Because the column type of the partition table must be the same as its parent
+# table, only test normal table here.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE DOMAIN test_domain AS int CHECK(VALUE > random());
+ALTER TABLE test_tab ALTER COLUMN a TYPE test_domain;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop domain constraint expression on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping domain constraint expression'
+);
+
+# It is not allowed that constraint expression contains a non-immutable function
+# on the subscriber side. Check the error reported by background worker in this
+# case.
+# First we check the constraint expression on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# Then we check the constraint expression on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# It is not allowed that foreign key on the subscriber side. Check the error
+# reported by background worker in this case.
+# First we check the foreign key on normal table.
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_f (a int primary key);
+ALTER TABLE test_tab ADD CONSTRAINT test_tabfk FOREIGN KEY(a) REFERENCES test_tab_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+# Then we check the foreign key on partition table.
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_partition_f (a int primary key);
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_patition_fk FOREIGN KEY(a) REFERENCES test_tab_partition_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+$node_subscriber->stop;
+$node_publisher->stop;
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 4137dc77b4..ae7cd99159 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2271,6 +2271,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelParallel
RelToCheck
RelToCluster
RelabelType
--
2.23.0.windows.1
[application/octet-stream] v15-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch (26.6K, ../../OS3PR01MB62755C6C9A75EB09F7218B589E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-v15-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch)
download | inline diff:
From 25d936d1190c4ba6ec4c6cacb72e96b73f85c598 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 15 Jun 2022 11:02:08 +0800
Subject: [PATCH v15 4/4] Retry to apply streaming xact only in apply worker
If the user sets the subscription_parameter "streaming" to "parallel", when
applying a streaming transaction, we will try to apply this transaction in
apply background worker. However, when the changes in this transaction cannot
be applied in apply background worker, the background worker will exit with an
error. In this case, we can retry applying this streaming transaction in "on"
mode. In this way, we may avoid blocking logical replication here.
So we introduce field "subretry" in catalog "pg_subscription". When the
subscriber exit with an error, we will try to set this flag to true, and when
the transaction is applied successfully, we will try to set this flag to false.
Then when we try to apply a streaming transaction in apply background worker,
we can see if this transaction has failed before based on the "subretry" field.
---
doc/src/sgml/catalogs.sgml | 9 +
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/catalog/pg_subscription.c | 1 +
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/subscriptioncmds.c | 1 +
.../replication/logical/applybgworker.c | 15 +-
src/backend/replication/logical/worker.c | 89 ++++++++++
src/bin/pg_dump/pg_dump.c | 5 +-
src/include/catalog/pg_subscription.h | 4 +
.../subscription/t/032_streaming_apply.pl | 154 +++++++++++-------
10 files changed, 221 insertions(+), 66 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 815cae6082..28f3d121d9 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7907,6 +7907,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>subretry</structfield> <type>bool</type>
+ </para>
+ <para>
+ True if the previous apply change failed and a retry was required.
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>subconninfo</structfield> <type>text</type>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 270e3d382e..bd5361991e 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -244,6 +244,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
relation on the subscriber-side should also be the unique column on
the publisher-side; 2) there cannot be any non-immutable functions
in the subscriber-side replicated table.
+ When applying a streaming transaction, if either requirement is not
+ met, the background worker will exit with an error.
+ <literal>parallel</literal> mode is disregarded when retrying;
+ instead the transaction will be applied using <literal>on</literal>
+ mode.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index 8856ce3b50..9b7f09653d 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok)
sub->stream = subform->substream;
sub->twophasestate = subform->subtwophasestate;
sub->disableonerr = subform->subdisableonerr;
+ sub->retry = subform->subretry;
/* Get conninfo */
datum = SysCacheGetAttr(SUBSCRIPTIONOID,
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fedaed533b..10f4dd6785 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1298,8 +1298,8 @@ REVOKE ALL ON pg_replication_origin_status FROM public;
-- All columns of pg_subscription except subconninfo are publicly readable.
REVOKE ALL ON pg_subscription FROM public;
GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled,
- subbinary, substream, subtwophasestate, subdisableonerr, subslotname,
- subsynccommit, subpublications)
+ subbinary, substream, subtwophasestate, subdisableonerr,
+ subretry, subslotname, subsynccommit, subpublications)
ON pg_subscription TO public;
CREATE VIEW pg_stat_subscription_stats AS
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 5f349067cc..33aef31b30 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -662,6 +662,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
LOGICALREP_TWOPHASE_STATE_PENDING :
LOGICALREP_TWOPHASE_STATE_DISABLED);
values[Anum_pg_subscription_subdisableonerr - 1] = BoolGetDatum(opts.disableonerr);
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(false);
values[Anum_pg_subscription_subconninfo - 1] =
CStringGetTextDatum(conninfo);
if (opts.slot_name)
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index 92b4b4e4ed..9bcc8669fb 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -106,6 +106,18 @@ apply_bgworker_can_start(TransactionId xid)
if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
return false;
+ /*
+ * Don't use apply background workers for retries, because it is possible
+ * that the last time we tried to apply a transaction using an apply
+ * background worker the checks failed (see function
+ * apply_bgworker_relation_check).
+ */
+ if (MySubscription->retry)
+ {
+ elog(DEBUG1, "apply background workers are not used for retries");
+ return false;
+ }
+
/*
* For streaming transactions that are being applied in apply background
* worker, we cannot decide whether to apply the change for a relation
@@ -812,6 +824,5 @@ apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
"mode", rel->remoterel.nspname, rel->remoterel.relname),
errdetail("The unique column on subscriber is not the unique "
"column on publisher or there is at least one "
- "non-immutable function."),
- errhint("Please change the streaming option to 'on' instead of 'parallel'.")));
+ "non-immutable function.")));
}
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 81e0005d12..a4aa6de533 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -378,6 +378,8 @@ static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
+static void set_subscription_retry(bool retry);
+
/*
* Should this worker apply changes for given relation.
*
@@ -904,6 +906,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1015,6 +1020,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1068,6 +1076,9 @@ apply_handle_commit_prepared(StringInfo s)
store_flush_position(prepare_data.end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1123,6 +1134,9 @@ apply_handle_rollback_prepared(StringInfo s)
store_flush_position(rollback_data.rollback_end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(rollback_data.rollback_end_lsn);
@@ -1215,6 +1229,9 @@ apply_handle_stream_prepare(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
in_remote_transaction = false;
@@ -1642,6 +1659,9 @@ apply_handle_stream_abort(StringInfo s)
*/
serialize_stream_abort(xid, subxid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
reset_apply_error_context_info();
@@ -1854,6 +1874,9 @@ apply_handle_stream_commit(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
/* Check the status of apply background worker if any. */
@@ -3897,6 +3920,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
}
PG_CATCH();
{
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
else
@@ -3935,6 +3961,9 @@ start_apply(XLogRecPtr origin_startpos)
}
PG_CATCH();
{
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
else
@@ -4461,3 +4490,63 @@ reset_apply_error_context_info(void)
apply_error_callback_arg.remote_attnum = -1;
set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
}
+
+/*
+ * Set subretry of pg_subscription catalog.
+ *
+ * If retry is true, subscriber is about to exit with an error. Otherwise, it
+ * means that the transaction was applied successfully.
+ */
+static void
+set_subscription_retry(bool retry)
+{
+ Relation rel;
+ HeapTuple tup;
+ bool started_tx = false;
+ bool nulls[Natts_pg_subscription];
+ bool replaces[Natts_pg_subscription];
+ Datum values[Natts_pg_subscription];
+
+ if (MySubscription->retry == retry ||
+ am_apply_bgworker())
+ return;
+
+ if (!IsTransactionState())
+ {
+ StartTransactionCommand();
+ started_tx = true;
+ }
+
+ /* Look up the subscription in the catalog */
+ rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+ tup = SearchSysCacheCopy1(SUBSCRIPTIONOID,
+ ObjectIdGetDatum(MySubscription->oid));
+
+ if (!HeapTupleIsValid(tup))
+ elog(ERROR, "subscription \"%s\" does not exist", MySubscription->name);
+
+ LockSharedObject(SubscriptionRelationId, MySubscription->oid, 0,
+ AccessShareLock);
+
+ /* Form a new tuple. */
+ memset(values, 0, sizeof(values));
+ memset(nulls, false, sizeof(nulls));
+ memset(replaces, false, sizeof(replaces));
+
+ /* reset subretry */
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(retry);
+ replaces[Anum_pg_subscription_subretry - 1] = true;
+
+ tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+ replaces);
+
+ /* Update the catalog. */
+ CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+ /* Cleanup. */
+ heap_freetuple(tup);
+ table_close(rel, NoLock);
+
+ if (started_tx)
+ CommitTransactionCommand();
+}
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 24927641b9..9ce774fc39 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4471,8 +4471,9 @@ getSubscriptions(Archive *fout)
ntups = PQntuples(res);
/*
- * Get subscription fields. We don't include subskiplsn in the dump as
- * after restoring the dump this value may no longer be relevant.
+ * Get subscription fields. We don't include subskiplsn and subretry in
+ * the dump as after restoring the dump this value may no longer be
+ * relevant.
*/
i_tableoid = PQfnumber(res, "tableoid");
i_oid = PQfnumber(res, "oid");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d54540f5f5..5f4e058ec1 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -76,6 +76,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subdisableonerr; /* True if a worker error should cause the
* subscription to be disabled */
+ bool subretry BKI_DEFAULT(f); /* True if the previous apply change failed. */
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* Connection string to the publisher */
text subconninfo BKI_FORCE_NOT_NULL;
@@ -116,6 +118,8 @@ typedef struct Subscription
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
* occurs */
+ bool retry; /* Indicates if the previous apply change
+ * failed. */
char *conninfo; /* Connection string to the publisher */
char *slotname; /* Name of the replication slot */
char *synccommit; /* Synchronous commit setting for worker */
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
index eca4328676..1bcb4c65b7 100644
--- a/src/test/subscription/t/032_streaming_apply.pl
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -57,8 +57,13 @@ $node_subscriber->safe_psql(
$node_publisher->wait_for_catchup($appname);
+# ============================================================================
# It is not allowed that the unique index on the publisher and the subscriber
-# is different. Check the error reported by background worker in this case.
+# is different. Check the error reported by background worker in this case. And
+# after retrying in apply worker, we check if the data is replicated
+# successfully.
+# ============================================================================
+
# First we check the unique index on normal table.
$node_subscriber->safe_psql('postgres',
"CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
@@ -82,14 +87,15 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
my $result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
# Then we check the unique index on partition table.
$node_subscriber->safe_psql('postgres',
@@ -106,17 +112,20 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
# Triggers which execute non-immutable function are not allowed on the
# subscriber side. Check the error reported by background worker in this case.
+# And after retrying in apply worker, we check if the data is replicated
+# successfully.
# First we check the trigger function on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -140,15 +149,16 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
# Then we check the trigger function on partition table.
$node_subscriber->safe_psql(
@@ -168,19 +178,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab_partition");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+# ============================================================================
# It is not allowed that column default value expression contains a
# non-immutable function on the subscriber side. Check the error reported by
-# background worker in this case.
+# background worker in this case. And after retrying in apply worker, we check
+# if the data is replicated successfully.
+# ============================================================================
+
# First we check the column default value expression on normal table.
$node_subscriber->safe_psql('postgres',
"ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
@@ -196,16 +211,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
# Then we check the column default value expression on partition table.
$node_subscriber->safe_psql('postgres',
@@ -222,20 +238,25 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+# ============================================================================
# It is not allowed that domain constraint expression contains a non-immutable
# function on the subscriber side. Check the error reported by background
-# worker in this case.
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# Because the column type of the partition table must be the same as its parent
# table, only test normal table here.
$node_subscriber->safe_psql(
@@ -253,21 +274,26 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop domain constraint expression on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(0),
- 'data replicated to subscriber after dropping domain constraint expression'
+ 'data replicated to subscribers after retrying because of domain'
);
-# It is not allowed that constraint expression contains a non-immutable function
-# on the subscriber side. Check the error reported by background worker in this
-# case.
+# Drop domain constraint expression on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+# ============================================================================
+# It is not allowed that constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# First we check the constraint expression on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -285,16 +311,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
# Then we check the constraint expression on partition table.
$node_subscriber->safe_psql(
@@ -311,19 +338,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(0),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+# ============================================================================
# It is not allowed that foreign key on the subscriber side. Check the error
-# reported by background worker in this case.
+# reported by background worker in this case. And after retrying in apply
+# worker, we check if the data is replicated successfully.
+# ============================================================================
+
# First we check the foreign key on normal table.
$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
$node_publisher->wait_for_catchup($appname);
@@ -344,16 +376,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
# Then we check the foreign key on partition table.
$node_publisher->wait_for_catchup($appname);
@@ -374,16 +407,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
$node_subscriber->stop;
$node_publisher->stop;
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-07 03:45 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: [email protected] @ 2022-07-07 03:45 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 1, 2022 at 17:44 PM Amit Kapila <[email protected]> wrote:
>
Thanks for your comments.
> On Fri, Jul 1, 2022 at 12:13 PM Peter Smith <[email protected]> wrote:
> >
> > ======
> >
> > 1.2 doc/src/sgml/protocol.sgml - Protocol constants
> >
> > Previously I wrote that since there are protocol changes here,
> > shouldn’t there also be some corresponding LOGICALREP_PROTO_XXX
> > constants and special checking added in the worker.c?
> >
> > But you said [1 comment #6] you think it is OK because...
> >
> > IMO, I still disagree with the reply. The fact is that the protocol
> > *has* been changed, so IIUC that is precisely the reason for having
> > those protocol constants.
> >
> > e.g I am guessing you might assign the new one somewhere here:
> > --
> > server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
> > options.proto.logical.proto_version =
> > server_version >= 150000 ?
> LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
> > server_version >= 140000 ?
> LOGICALREP_PROTO_STREAM_VERSION_NUM :
> > LOGICALREP_PROTO_VERSION_NUM;
> > --
> >
> > And then later you would refer to this new protocol version (instead
> > of the server version) when calling to the apply_handle_stream_abort
> > function.
> >
> > ======
> >
>
> One point related to this that occurred to me is how it will behave if
> the publisher is of version >=16 whereas the subscriber is of versions
> <=15? Won't in that case publisher sends the new fields but
> subscribers won't be reading those which may cause some problems.
Makes sense. Fixed this point.
As Peter-san suggested, I added a new protocol macro
LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM.
This new macro marks the version that supports apply background worker (it
means we will read abort_lsn and abort_time). And the publisher sends abort_lsn
and abort_time fields only if subscriber will read them. (see function
logicalrep_write_stream_abort)
The new patches were attached in [1].
[1] - https://www.postgresql.org/message-id/OS3PR01MB62755C6C9A75EB09F7218B589E839%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-07 10:20 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 43+ messages in thread
From: [email protected] @ 2022-07-07 10:20 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 7, 2022 at 11:44 AM I wrote:
> Attach the new patches.
I found a failure on CFbot [1], which after investigation I think is due to my
previous modification (see response to #1.10 in [2]).
For a streaming transaction, if we failed in the first chunk of streamed
changes for this transaction in the apply background worker, we will set the
status of this apply background worker to APPLY_BGWORKER_EXIT.
And at the same time, main apply worker obtains apply background worker
in the function apply_bgworker_find when processing the second chunk of
streamed changes for this transaction, the status of apply background worker
is APPLY_BGWORKER_EXIT. So the following assertion will fail:
```
Assert(status == APPLY_BGWORKER_BUSY);
```
To fix this, before invoking function assert, I try to detect the failure of
apply background worker. If the status is APPLY_BGWORKER_EXIT, then exit with
an error.
I also made some other small improvements.
Attach the new patches.
[1] - https://cirrus-ci.com/task/6383178511286272?logs=test_world#L2636
[2] - https://www.postgresql.org/message-id/OS3PR01MB62755C6C9A75EB09F7218B589E839%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
Attachments:
[application/octet-stream] v16-0001-Perform-streaming-logical-transactions-by-backgr.patch (105.9K, ../../OS3PR01MB627594D75E870BBB45E2E80A9E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-v16-0001-Perform-streaming-logical-transactions-by-backgr.patch)
download | inline diff:
From 4d3bc1a5ba7b15f6d5fd0a975912052fcf124604 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Wed, 20 Apr 2022 16:45:07 +0800
Subject: [PATCH v16 1/4] Perform streaming logical transactions by background
workers
Currently, for large transactions, the publisher sends the data in multiple
streams (changes divided into chunks depending upon logical_decoding_work_mem),
and then on the subscriber-side, the apply worker writes the changes into
temporary files and once it receives the commit, it reads from the file and
applies the entire transaction. To improve the performance of such
transactions, we can instead allow them to be applied via background workers.
In this approach, we assign a new apply background worker (if available) as
soon as the xact's first stream is received and the main apply worker will send
changes to this new worker via shared memory. The apply background worker will
directly apply the change instead of writing it to temporary files. We keep
this worker assigned till the transaction commit is received and also wait for
the worker to finish at commit. This preserves commit ordering and avoids
writing to and reading from file in most cases. We still need to spill if there
is no worker available.
This patch also extends the SUBSCRIPTION 'streaming' parameter so that the user
can control whether to apply the streaming transaction in an apply background
worker or spill the change to disk. The user can set the streaming parameter to
'on/off', 'parallel'. The parameter value 'parallel' means the streaming will
be applied via an apply background worker, if available. The parameter value
'on' means the streaming transaction will be spilled to disk. The default value
is 'off' (same as current behaviour).
---
doc/src/sgml/catalogs.sgml | 10 +-
doc/src/sgml/config.sgml | 25 +
doc/src/sgml/logical-replication.sgml | 10 +
doc/src/sgml/protocol.sgml | 19 +
doc/src/sgml/ref/create_subscription.sgml | 24 +-
src/backend/access/transam/xact.c | 13 +
src/backend/commands/subscriptioncmds.c | 66 +-
src/backend/postmaster/bgworker.c | 3 +
src/backend/replication/logical/Makefile | 1 +
.../replication/logical/applybgworker.c | 786 ++++++++++++++++++
src/backend/replication/logical/decode.c | 10 +-
src/backend/replication/logical/launcher.c | 130 ++-
src/backend/replication/logical/origin.c | 26 +-
src/backend/replication/logical/proto.c | 41 +-
.../replication/logical/reorderbuffer.c | 10 +-
src/backend/replication/logical/tablesync.c | 10 +-
src/backend/replication/logical/worker.c | 688 +++++++++++----
src/backend/replication/pgoutput/pgoutput.c | 9 +-
src/backend/utils/activity/wait_event.c | 3 +
src/backend/utils/misc/guc.c | 12 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/bin/pg_dump/pg_dump.c | 6 +-
src/include/catalog/pg_subscription.h | 21 +-
src/include/replication/logicallauncher.h | 1 +
src/include/replication/logicalproto.h | 27 +-
src/include/replication/logicalworker.h | 1 +
src/include/replication/origin.h | 2 +-
src/include/replication/reorderbuffer.h | 7 +-
src/include/replication/worker_internal.h | 102 ++-
src/include/utils/wait_event.h | 1 +
src/test/regress/expected/subscription.out | 2 +-
src/tools/pgindent/typedefs.list | 5 +
32 files changed, 1852 insertions(+), 220 deletions(-)
create mode 100644 src/backend/replication/logical/applybgworker.c
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 4f3f375a84..815cae6082 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7873,11 +7873,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry role="catalog_table_entry"><para role="column_definition">
- <structfield>substream</structfield> <type>bool</type>
+ <structfield>substream</structfield> <type>char</type>
</para>
<para>
- If true, the subscription will allow streaming of in-progress
- transactions
+ Controls how to handle the streaming of in-progress transactions:
+ <literal>f</literal> = disallow streaming of in-progress transactions,
+ <literal>t</literal> = spill the changes of in-progress transactions to
+ disk and apply at once after the transaction is committed on the
+ publisher,
+ <literal>p</literal> = apply changes directly using a background worker
</para></entry>
</row>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..6bbd986195 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4970,6 +4970,31 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
</listitem>
</varlistentry>
+ <varlistentry id="guc-max-apply-bgworkers-per-subscription" xreflabel="max_apply_bgworkers_per_subscription">
+ <term><varname>max_apply_bgworkers_per_subscription</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>max_apply_bgworkers_per_subscription</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Maximum number of apply background workers per subscription. This
+ parameter controls the amount of parallelism of the streaming of
+ in-progress transactions when subscription parameter
+ <literal>streaming = parallel</literal>.
+ </para>
+ <para>
+ The apply background workers are taken from the pool defined by
+ <varname>max_logical_replication_workers</varname>.
+ </para>
+ <para>
+ The default value is 2. This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command
+ line.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</sect2>
diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml
index bdf1e7b727..92997f9299 100644
--- a/doc/src/sgml/logical-replication.sgml
+++ b/doc/src/sgml/logical-replication.sgml
@@ -1153,6 +1153,16 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER
might not violate any constraint. This can easily make the subscriber
inconsistent.
</para>
+
+ <para>
+ When the streaming mode is <literal>parallel</literal>, the finish LSN of
+ failed transactions may not be logged. In that case, it may be necessary to
+ change the streaming mode to <literal>on</literal> and cause the same
+ conflicts again so the finish LSN of the failed transaction will be written
+ to the server log. For the usage of finish LSN, please refer to <link
+ linkend="sql-altersubscription"><command>ALTER SUBSCRIPTION ...
+ SKIP</command></link>.
+ </para>
</sect1>
<sect1 id="logical-replication-restrictions">
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..7e88ba9631 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -6809,6 +6809,25 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term>Int64 (XLogRecPtr)</term>
+ <listitem>
+ <para>
+ The LSN of the abort.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>Int64 (TimestampTz)</term>
+ <listitem>
+ <para>
+ Abort timestamp of the transaction. The value is in number
+ of microseconds since PostgreSQL epoch (2000-01-01).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term>Int32 (TransactionId)</term>
<listitem>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 34b3264b26..71dd4aca81 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -217,13 +217,29 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</varlistentry>
<varlistentry>
- <term><literal>streaming</literal> (<type>boolean</type>)</term>
+ <term><literal>streaming</literal> (<type>enum</type>)</term>
<listitem>
<para>
Specifies whether to enable streaming of in-progress transactions
- for this subscription. By default, all transactions
- are fully decoded on the publisher and only then sent to the
- subscriber as a whole.
+ for this subscription. The default value is <literal>off</literal>,
+ meaning all transactions are fully decoded on the publisher and only
+ then sent to the subscriber as a whole.
+ </para>
+
+ <para>
+ If set to <literal>on</literal>, the incoming changes are written to
+ temporary files and then applied only after the transaction is
+ committed on the publisher.
+ </para>
+
+ <para>
+ If set to <literal>parallel</literal>, incoming changes are directly
+ applied via one of the apply background workers, if available. If no
+ background worker is free to handle streaming transaction then the
+ changes are written to temporary files and applied after the
+ transaction is committed. Note that if an error happens when
+ applying changes in a background worker, the finish LSN of the
+ remote transaction might not be reported in the server log.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 116de1175b..3e61a57b50 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -1711,6 +1711,7 @@ RecordTransactionAbort(bool isSubXact)
int nchildren;
TransactionId *children;
TimestampTz xact_time;
+ bool replorigin;
/*
* If we haven't been assigned an XID, nobody will care whether we aborted
@@ -1741,6 +1742,13 @@ RecordTransactionAbort(bool isSubXact)
elog(PANIC, "cannot abort transaction %u, it was already committed",
xid);
+ /*
+ * Are we using the replication origins feature? Or, in other words,
+ * are we replaying remote actions?
+ */
+ replorigin = (replorigin_session_origin != InvalidRepOriginId &&
+ replorigin_session_origin != DoNotReplicateId);
+
/* Fetch the data we need for the abort record */
nrels = smgrGetPendingDeletes(false, &rels);
nchildren = xactGetCommittedChildren(&children);
@@ -1765,6 +1773,11 @@ RecordTransactionAbort(bool isSubXact)
MyXactFlags, InvalidTransactionId,
NULL);
+ if (replorigin)
+ /* Move LSNs forward for this replication origin */
+ replorigin_session_advance(replorigin_session_origin_lsn,
+ XactLastRecEnd);
+
/*
* Report the latest async abort LSN, so that the WAL writer knows to
* flush this abort. There's nothing to be gained by delaying this, since
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bdc1208724..5f349067cc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -83,7 +83,7 @@ typedef struct SubOpts
bool copy_data;
bool refresh;
bool binary;
- bool streaming;
+ char streaming;
bool twophase;
bool disableonerr;
XLogRecPtr lsn;
@@ -95,6 +95,62 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub,
static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
+/*
+ * Extract the streaming mode value from a DefElem. This is like
+ * defGetBoolean() but also accepts the special value of "parallel".
+ */
+static char
+defGetStreamingMode(DefElem *def)
+{
+ /*
+ * If no value given, assume "true" is meant.
+ */
+ if (def->arg == NULL)
+ return SUBSTREAM_ON;
+
+ /*
+ * Allow 0, 1, "false", "true", "off", "on" or "parallel".
+ */
+ switch (nodeTag(def->arg))
+ {
+ case T_Integer:
+ switch (intVal(def->arg))
+ {
+ case 0:
+ return SUBSTREAM_OFF;
+ case 1:
+ return SUBSTREAM_ON;
+ default:
+ /* otherwise, error out below */
+ break;
+ }
+ break;
+ default:
+ {
+ char *sval = defGetString(def);
+
+ /*
+ * The set of strings accepted here should match up with the
+ * grammar's opt_boolean_or_string production.
+ */
+ if (pg_strcasecmp(sval, "false") == 0 ||
+ pg_strcasecmp(sval, "off") == 0)
+ return SUBSTREAM_OFF;
+ if (pg_strcasecmp(sval, "true") == 0 ||
+ pg_strcasecmp(sval, "on") == 0)
+ return SUBSTREAM_ON;
+ if (pg_strcasecmp(sval, "parallel") == 0)
+ return SUBSTREAM_PARALLEL;
+ }
+ break;
+ }
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("%s requires a Boolean value or \"parallel\"",
+ def->defname)));
+ return SUBSTREAM_OFF; /* keep compiler quiet */
+}
+
/*
* Common option parsing function for CREATE and ALTER SUBSCRIPTION commands.
*
@@ -132,7 +188,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
if (IsSet(supported_opts, SUBOPT_BINARY))
opts->binary = false;
if (IsSet(supported_opts, SUBOPT_STREAMING))
- opts->streaming = false;
+ opts->streaming = SUBSTREAM_OFF;
if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
opts->twophase = false;
if (IsSet(supported_opts, SUBOPT_DISABLE_ON_ERR))
@@ -233,7 +289,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
errorConflictingDefElem(defel, pstate);
opts->specified_opts |= SUBOPT_STREAMING;
- opts->streaming = defGetBoolean(defel);
+ opts->streaming = defGetStreamingMode(defel);
}
else if (strcmp(defel->defname, "two_phase") == 0)
{
@@ -600,7 +656,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner);
values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled);
values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary);
- values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming);
+ values[Anum_pg_subscription_substream - 1] = CharGetDatum(opts.streaming);
values[Anum_pg_subscription_subtwophasestate - 1] =
CharGetDatum(opts.twophase ?
LOGICALREP_TWOPHASE_STATE_PENDING :
@@ -1059,7 +1115,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
if (IsSet(opts.specified_opts, SUBOPT_STREAMING))
{
values[Anum_pg_subscription_substream - 1] =
- BoolGetDatum(opts.streaming);
+ CharGetDatum(opts.streaming);
replaces[Anum_pg_subscription_substream - 1] = true;
}
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 40601aefd9..40ccb8993c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -128,6 +128,9 @@ static const struct
},
{
"ApplyWorkerMain", ApplyWorkerMain
+ },
+ {
+ "ApplyBgworkerMain", ApplyBgworkerMain
}
};
diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile
index c4e2fdeb71..cbfb5d794e 100644
--- a/src/backend/replication/logical/Makefile
+++ b/src/backend/replication/logical/Makefile
@@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(srcdir) $(CPPFLAGS)
OBJS = \
+ applybgworker.o \
decode.o \
launcher.o \
logical.o \
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
new file mode 100644
index 0000000000..bb2b180e29
--- /dev/null
+++ b/src/backend/replication/logical/applybgworker.c
@@ -0,0 +1,786 @@
+/*-------------------------------------------------------------------------
+ * applybgworker.c
+ * Support routines for applying xact by apply background worker
+ *
+ * Copyright (c) 2016-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/logical/applybgworker.c
+ *
+ * This file contains routines that are intended to support setting up, using,
+ * and tearing down a ApplyBgworkerState.
+ *
+ * Refer to the comments in file header of logical/worker.c to see more
+ * information about apply background worker.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "libpq/pqformat.h"
+#include "mb/pg_wchar.h"
+#include "pgstat.h"
+#include "postmaster/interrupt.h"
+#include "replication/logicallauncher.h"
+#include "replication/logicalworker.h"
+#include "replication/origin.h"
+#include "replication/walreceiver.h"
+#include "replication/worker_internal.h"
+#include "storage/ipc.h"
+#include "storage/procarray.h"
+#include "tcop/tcopprot.h"
+#include "utils/inval.h"
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+#include "utils/syscache.h"
+
+#define PG_LOGICAL_APPLY_SHM_MAGIC 0x79fb2447 // TODO Consider change
+
+/*
+ * DSM keys for apply background worker. Unlike other parallel execution code,
+ * since we don't need to worry about DSM keys conflicting with plan_node_id we
+ * can use small integers.
+ */
+#define APPLY_BGWORKER_KEY_SHARED 1
+#define APPLY_BGWORKER_KEY_MQ 2
+
+/* Queue size of DSM, 16 MB for now. */
+#define DSM_QUEUE_SIZE 160000000
+
+/*
+ * There are three fields in message: start_lsn, end_lsn and send_time. Because
+ * we have updated these statistics in apply worker, we could ignore these
+ * fields in apply background worker. (see function LogicalRepApplyLoop)
+ */
+#define IGNORE_SIZE_IN_MESSAGE (3 * sizeof(uint64))
+
+/*
+ * Entry for a hash table we use to map from xid to our apply background worker
+ * state.
+ */
+typedef struct ApplyBgworkerEntry
+{
+ TransactionId xid;
+ ApplyBgworkerState *wstate;
+} ApplyBgworkerEntry;
+
+/* Apply background workers hash table (initialized on first use) */
+static HTAB *ApplyWorkersHash = NULL;
+static List *ApplyWorkersFreeList = NIL;
+static List *ApplyWorkersList = NIL;
+
+/*
+ * Information shared between main apply worker and apply background worker.
+ */
+volatile ApplyBgworkerShared *MyParallelState = NULL;
+
+List *subxactlist = NIL;
+
+static bool apply_bgworker_can_start(TransactionId xid);
+static ApplyBgworkerState *apply_bgworker_setup(void);
+static void apply_bgworker_setup_dsm(ApplyBgworkerState *wstate);
+
+/*
+ * Check if starting a new apply background worker is allowed.
+ */
+static bool
+apply_bgworker_can_start(TransactionId xid)
+{
+ if (!TransactionIdIsValid(xid))
+ return false;
+
+ /*
+ * Don't start a new background worker if not in streaming parallel mode.
+ */
+ if (MySubscription->stream != SUBSTREAM_PARALLEL)
+ return false;
+
+ /*
+ * Don't start a new background worker if user has set skiplsn as it's
+ * possible that user want to skip the streaming transaction. For
+ * streaming transaction, we need to spill the transaction to disk so that
+ * we can get the last LSN of the transaction to judge whether to skip
+ * before starting to apply the change.
+ */
+ if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
+ return false;
+
+ /*
+ * For streaming transactions that are being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time. So, we don't start new apply
+ * background worker in this case.
+ */
+ if (!AllTablesyncsReady())
+ return false;
+
+ return true;
+}
+
+/*
+ * Try to start an apply background worker and, if successful, cache it in
+ * ApplyWorkersHash keyed by the specified xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_start(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerState *wstate;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!apply_bgworker_can_start(xid))
+ return NULL;
+
+ /* First time through, initialize apply workers hashtable */
+ if (ApplyWorkersHash == NULL)
+ {
+ HASHCTL ctl;
+
+ MemSet(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(TransactionId);
+ ctl.entrysize = sizeof(ApplyBgworkerEntry);
+ ctl.hcxt = ApplyContext;
+
+ ApplyWorkersHash = hash_create("logical apply workers hash", 8, &ctl,
+ HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+ }
+
+ /*
+ * Now, we try to get an apply background worker. If there is at least one
+ * worker in the free list, then take one. Otherwise, we try to start a
+ * new apply background worker.
+ */
+ if (list_length(ApplyWorkersFreeList) > 0)
+ {
+ wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
+ ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
+ Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ wstate = apply_bgworker_setup();
+
+ if (wstate == NULL)
+ return NULL;
+ }
+
+ /*
+ * Create entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_ENTER, &found);
+ if (found)
+ elog(ERROR, "hash table corrupted");
+
+ /* Fill up the hash entry */
+ wstate->pstate->status = APPLY_BGWORKER_BUSY;
+ wstate->pstate->server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ wstate->pstate->stream_xid = xid;
+ entry->wstate = wstate;
+ entry->xid = xid;
+
+ return wstate;
+}
+
+/*
+ * Try to look up worker inside ApplyWorkersHash for requested xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_find(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!TransactionIdIsValid(xid))
+ return NULL;
+
+ if (ApplyWorkersHash == NULL)
+ return NULL;
+
+ /*
+ * Find entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
+ if (found)
+ {
+ char status = entry->wstate->pstate->status;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ entry->wstate->pstate->n,
+ entry->wstate->pstate->stream_xid)));
+
+ Assert(status == APPLY_BGWORKER_BUSY);
+
+ return entry->wstate;
+ }
+ else
+ return NULL;
+}
+
+/*
+ * Add the worker to the free list and remove the entry from the hash table.
+ */
+void
+apply_bgworker_free(ApplyBgworkerState *wstate)
+{
+ MemoryContext oldctx;
+ TransactionId xid = wstate->pstate->stream_xid;
+
+ Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ hash_search(ApplyWorkersHash, &xid, HASH_REMOVE, NULL);
+
+ elog(DEBUG1, "adding finished apply worker #%u for xid %u to the free list",
+ wstate->pstate->n, wstate->pstate->stream_xid);
+
+ ApplyWorkersFreeList = lappend(ApplyWorkersFreeList, wstate);
+
+ MemoryContextSwitchTo(oldctx);
+}
+
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *pst)
+{
+ shm_mq_result shmq_res;
+ PGPROC *registrant;
+ ErrorContextCallback errcallback;
+
+ registrant = BackendPidGetProc(MyBgworkerEntry->bgw_notify_pid);
+ SetLatch(®istrant->procLatch);
+
+ /*
+ * Push apply error context callback. Fields will be filled applying a
+ * change.
+ */
+ errcallback.callback = apply_error_callback;
+ errcallback.previous = error_context_stack;
+ error_context_stack = &errcallback;
+
+ for (;;)
+ {
+ void *data;
+ Size len;
+ int c;
+ StringInfoData s;
+ MemoryContext oldctx;
+
+ CHECK_FOR_INTERRUPTS();
+
+ /* Ensure we are reading the data into our memory context. */
+ oldctx = MemoryContextSwitchTo(ApplyMessageContext);
+
+ shmq_res = shm_mq_receive(mqh, &len, &data, false);
+
+ if (shmq_res != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("lost connection to the main apply worker")));
+
+ if (len == 0)
+ break;
+
+ s.cursor = 0;
+ s.maxlen = -1;
+ s.data = (char *) data;
+ s.len = len;
+
+ /*
+ * We use first byte of message for additional communication between
+ * main Logical replication worker and apply background workers, so if
+ * it differs from 'w', then process it first.
+ */
+ c = pq_getmsgbyte(&s);
+ switch (c)
+ {
+ /* End message of streaming chunk */
+ case LOGICAL_REP_MSG_STREAM_STOP:
+ elog(DEBUG1, "[Apply BGW #%u] ended processing streaming chunk,"
+ "waiting on shm_mq_receive", pst->n);
+
+ in_streamed_transaction = false;
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ continue;
+ case 'w':
+ break;
+ default:
+ elog(ERROR, "[Apply BGW #%u] unexpected message \"%c\"",
+ pst->n, c);
+ break;
+ }
+
+ /* Ignore statistics fields that have been updated. */
+ s.cursor += IGNORE_SIZE_IN_MESSAGE;
+
+ apply_dispatch(&s);
+
+ if (ConfigReloadPending)
+ {
+ ConfigReloadPending = false;
+ ProcessConfigFile(PGC_SIGHUP);
+ }
+
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
+ }
+
+ MemoryContextSwitchTo(TopMemoryContext);
+ MemoryContextReset(ApplyContext);
+
+ /* Pop the error context stack */
+ error_context_stack = errcallback.previous;
+
+ elog(DEBUG1, "[Apply BGW #%u] exiting", pst->n);
+
+ /* Signal main process that we are done. */
+ SetLatch(®istrant->procLatch);
+}
+
+/*
+ * Set the exit status so that the main apply worker can realize we have
+ * shutdown.
+ */
+static void
+apply_bgworker_shutdown(int code, Datum arg)
+{
+ SpinLockAcquire(&MyParallelState->mutex);
+ MyParallelState->status = APPLY_BGWORKER_EXIT;
+ SpinLockRelease(&MyParallelState->mutex);
+
+ dsm_detach((dsm_segment *) DatumGetPointer(arg));
+}
+
+/*
+ * Apply Background Worker entry point
+ */
+void
+ApplyBgworkerMain(Datum main_arg)
+{
+ volatile ApplyBgworkerShared *pst;
+
+ dsm_handle handle;
+ dsm_segment *seg;
+ shm_toc *toc;
+ shm_mq *mq;
+ shm_mq_handle *mqh;
+ MemoryContext oldcontext;
+ RepOriginId originid;
+ int worker_slot = DatumGetInt32(main_arg);
+ char originname[NAMEDATALEN];
+
+ MemoryContextSwitchTo(TopMemoryContext);
+
+ /* Init the memory context for the apply background worker to work in. */
+ ApplyContext = AllocSetContextCreate(TopMemoryContext,
+ "ApplyContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Init the ApplyMessageContext which we clean up after each replication
+ * protocol message.
+ */
+ ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+ "ApplyMessageContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /* Setup signal handling */
+ pqsignal(SIGHUP, SignalHandlerForConfigReload);
+ pqsignal(SIGTERM, die);
+ BackgroundWorkerUnblockSignals();
+
+ /*
+ * Connect to the dynamic shared memory segment.
+ *
+ * The backend that registered this worker passed us the ID of a shared
+ * memory segment to which we must attach for further instructions. In
+ * order to attach to dynamic shared memory, we need a resource owner.
+ * Once we've mapped the segment in our address space, attach to the table
+ * of contents so we can locate the various data structures we'll need to
+ * find within the segment.
+ */
+ CurrentResourceOwner = ResourceOwnerCreate(NULL, "Logical apply worker");
+ memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsm_handle));
+ seg = dsm_attach(handle);
+ if (seg == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("unable to map dynamic shared memory segment")));
+ toc = shm_toc_attach(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg));
+ if (toc == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("bad magic number in dynamic shared memory segment")));
+
+ before_shmem_exit(apply_bgworker_shutdown, PointerGetDatum(seg));
+
+ /* Look up the parallel state. */
+ pst = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_SHARED, false);
+ MyParallelState = pst;
+
+ /*
+ * Attach to the message queue.
+ */
+ mq = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_MQ, false);
+ shm_mq_set_receiver(mq, MyProc);
+ mqh = shm_mq_attach(mq, seg, NULL);
+
+ /* Run as replica session replication role. */
+ SetConfigOption("session_replication_role", "replica",
+ PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Now, we have initialized DSM. Attach to slot.
+ */
+ logicalrep_worker_attach(worker_slot);
+
+ /* Connect to our database. */
+ BackgroundWorkerInitializeConnectionByOid(MyLogicalRepWorker->dbid,
+ MyLogicalRepWorker->userid,
+ 0);
+
+ /*
+ * Set always-secure search path, so malicious users can't redirect user
+ * code (e.g. pg_index.indexprs).
+ */
+ SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Set the client encoding to the database encoding, since that is what
+ * the leader will expect.
+ */
+ SetClientEncoding(GetDatabaseEncoding());
+
+ stream_xid = pst->stream_xid;
+
+ StartTransactionCommand();
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ MySubscription = GetSubscription(MyLogicalRepWorker->subid, true);
+ if (!MySubscription)
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply worker for subscription %u will not "
+ "start because the subscription was removed during startup",
+ MyLogicalRepWorker->subid)));
+ proc_exit(0);
+ }
+
+ MySubscriptionValid = true;
+ MemoryContextSwitchTo(oldcontext);
+
+ /* Setup synchronous commit according to the user's wishes */
+ SetConfigOption("synchronous_commit", MySubscription->synccommit,
+ PGC_BACKEND, PGC_S_OVERRIDE);
+
+ /* Keep us informed about subscription changes. */
+ CacheRegisterSyscacheCallback(SUBSCRIPTIONOID,
+ subscription_change_cb,
+ (Datum) 0);
+
+ CommitTransactionCommand();
+
+ /* Setup replication origin tracking. */
+ StartTransactionCommand();
+ snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
+ originid = replorigin_by_name(originname, true);
+ if (!OidIsValid(originid))
+ originid = replorigin_create(originname);
+
+ /*
+ * The apply background worker doesn't need to monopolize this replication
+ * origin which was already acquired by its leader process.
+ */
+ replorigin_session_setup(originid, false);
+ replorigin_session_origin = originid;
+ CommitTransactionCommand();
+
+ /*
+ * Allocate the origin name in long-lived context for error context
+ * message.
+ */
+ apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+ originname);
+
+ elog(DEBUG1, "[Apply BGW #%u] started", pst->n);
+
+ LogicalApplyBgwLoop(mqh, pst);
+
+ /*
+ * We're done. Explicitly detach the shared memory segment so that we
+ * don't get a resource leak warning at commit time. This will fire any
+ * on_dsm_detach callbacks we've registered, as well. Once that's done,
+ * we can go ahead and exit.
+ */
+ dsm_detach(seg);
+ proc_exit(0);
+}
+
+/*
+ * Set up a dynamic shared memory segment.
+ *
+ * We set up a control region that contains a ApplyBgworkerShared,
+ * plus one region per message queue. There are as many message queues as
+ * the number of workers.
+ */
+static void
+apply_bgworker_setup_dsm(ApplyBgworkerState *wstate)
+{
+ shm_toc_estimator e;
+ Size segsize;
+ dsm_segment *seg;
+ shm_toc *toc;
+ ApplyBgworkerShared *pst;
+ shm_mq *mq;
+ int64 queue_size = DSM_QUEUE_SIZE;
+
+ /*
+ * Estimate how much shared memory we need.
+ *
+ * Because the TOC machinery may choose to insert padding of oddly-sized
+ * requests, we must estimate each chunk separately.
+ *
+ * We need one key to register the location of the header, and we need
+ * another key to track the location of the message queue.
+ */
+ shm_toc_initialize_estimator(&e);
+ shm_toc_estimate_chunk(&e, sizeof(ApplyBgworkerShared));
+ shm_toc_estimate_chunk(&e, (Size) queue_size);
+
+ shm_toc_estimate_keys(&e, 1 + 1);
+ segsize = shm_toc_estimate(&e);
+
+ /* Create the shared memory segment and establish a table of contents. */
+ seg = dsm_create(shm_toc_estimate(&e), 0);
+ toc = shm_toc_create(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg),
+ segsize);
+
+ /* Set up the header region. */
+ pst = shm_toc_allocate(toc, sizeof(ApplyBgworkerShared));
+ SpinLockInit(&pst->mutex);
+ pst->status = APPLY_BGWORKER_BUSY;
+ pst->server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ pst->stream_xid = stream_xid;
+ pst->n = list_length(ApplyWorkersList) + 1;
+
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_SHARED, pst);
+
+ /* Set up message queue for the worker. */
+ mq = shm_mq_create(shm_toc_allocate(toc, (Size) queue_size),
+ (Size) queue_size);
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_MQ, mq);
+ shm_mq_set_sender(mq, MyProc);
+
+ /* Attach the queue. */
+ wstate->mq_handle = shm_mq_attach(mq, seg, NULL);
+
+ /* Return results to caller. */
+ wstate->dsm_seg = seg;
+ wstate->pstate = pst;
+}
+
+/*
+ * Start apply background worker process and allocate shared memory for it.
+ */
+static ApplyBgworkerState *
+apply_bgworker_setup(void)
+{
+ MemoryContext oldcontext;
+ bool launched;
+ ApplyBgworkerState *wstate;
+ int napplyworkers;
+
+ elog(DEBUG1, "setting up apply worker #%u", list_length(ApplyWorkersList) + 1);
+
+ /* Check If there are free worker slot(s) */
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+ napplyworkers = logicalrep_apply_bgworker_count(MyLogicalRepWorker->subid);
+ LWLockRelease(LogicalRepWorkerLock);
+ if (napplyworkers >= max_apply_bgworkers_per_subscription)
+ return NULL;
+
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ wstate = (ApplyBgworkerState *) palloc0(sizeof(ApplyBgworkerState));
+
+ /* Setup shared memory */
+ apply_bgworker_setup_dsm(wstate);
+
+ launched = logicalrep_worker_launch(MyLogicalRepWorker->dbid,
+ MySubscription->oid,
+ MySubscription->name,
+ MyLogicalRepWorker->userid,
+ InvalidOid,
+ dsm_segment_handle(wstate->dsm_seg));
+
+ if (launched)
+ ApplyWorkersList = lappend(ApplyWorkersList, wstate);
+ else
+ {
+ dsm_detach(wstate->dsm_seg);
+ wstate->dsm_seg = NULL;
+
+ pfree(wstate);
+ wstate = NULL;
+ }
+
+ MemoryContextSwitchTo(oldcontext);
+
+ return wstate;
+}
+
+/*
+ * Send the data to the specified apply background worker via shared-memory queue.
+ */
+void
+apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes, const void *data)
+{
+ shm_mq_result result;
+
+ result = shm_mq_send(wstate->mq_handle, nbytes, data, false, true);
+
+ if (result != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("could not send tuples to shared-memory queue")));
+}
+
+/*
+ * Wait until the status of apply background worker reaches the
+ * 'wait_for_status'
+ */
+void
+apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status)
+{
+ for (;;)
+ {
+ char status;
+
+ SpinLockAcquire(&wstate->pstate->mutex);
+ status = wstate->pstate->status;
+ SpinLockRelease(&wstate->pstate->mutex);
+
+ /* Done if already in correct status. */
+ if (status == wait_for_status)
+ break;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ wstate->pstate->n, wstate->pstate->stream_xid)));
+
+ /* Wait to be signalled. */
+ WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 0,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE);
+
+ /* Reset the latch so we don't spin. */
+ ResetLatch(MyLatch);
+
+ /* An interrupt may have occurred while we were waiting. */
+ CHECK_FOR_INTERRUPTS();
+ }
+}
+
+/*
+ * Check the status of workers and report an error if any apply background
+ * worker has exited unexpectedly.
+ */
+void
+apply_bgworker_check_status(void)
+{
+ ListCell *lc;
+
+ if (am_apply_bgworker() || MySubscription->stream != SUBSTREAM_PARALLEL)
+ return;
+
+ foreach(lc, ApplyWorkersList)
+ {
+ ApplyBgworkerState *wstate = (ApplyBgworkerState *) lfirst(lc);
+
+ /*
+ * We don't lock here as in the worst case we will just detect the
+ * failure of worker a bit later.
+ */
+ if (wstate->pstate->status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u exited unexpectedly",
+ wstate->pstate->n)));
+ }
+
+ /*
+ * Exit if any relation is not in the READY state and if any worker is
+ * handling the streaming transaction at the same time. Because for
+ * streaming transactions that is being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time.
+ */
+ if (list_length(ApplyWorkersFreeList) != list_length(ApplyWorkersList) &&
+ !AllTablesyncsReady())
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply workers for subscription \"%s\" will restart",
+ MySubscription->name),
+ errdetail("Cannot handle streamed replication transaction by apply "
+ "background workers until all tables are synchronized")));
+
+ proc_exit(0);
+ }
+}
+
+/* Set the apply background worker status */
+void
+apply_bgworker_set_status(ApplyBgworkerStatus status)
+{
+ if (!am_apply_bgworker())
+ return;
+
+ elog(DEBUG1, "[Apply BGW #%u] set status to %d", MyParallelState->n, status);
+
+ SpinLockAcquire(&MyParallelState->mutex);
+ MyParallelState->status = status;
+ SpinLockRelease(&MyParallelState->mutex);
+}
+
+/*
+ * Define a savepoint for a subxact in apply background worker if needed.
+ *
+ * Inside apply background worker we can figure out that new subtransaction was
+ * started if new change arrived with different xid. In that case we can define
+ * named savepoint, so that we were able to commit/rollback it separately
+ * later.
+ * Special case is if the first change comes from subtransaction, then
+ * we check that current_xid differs from stream_xid.
+ */
+void
+apply_bgworker_subxact_info_add(TransactionId current_xid)
+{
+ if (current_xid != stream_xid &&
+ !list_member_int(subxactlist, (int) current_xid))
+ {
+ MemoryContext oldctx;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", current_xid);
+
+ elog(DEBUG1, "[Apply BGW #%u] defining savepoint %s",
+ MyParallelState->n, spname);
+
+ DefineSavepoint(spname);
+ CommitTransactionCommand();
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+ subxactlist = lappend_int(subxactlist, (int) current_xid);
+ MemoryContextSwitchTo(oldctx);
+ }
+}
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c5c6a2ba68..d4d5093a0b 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -651,9 +651,10 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
{
for (i = 0; i < parsed->nsubxacts; i++)
{
- ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr);
+ ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr,
+ commit_time);
}
- ReorderBufferForget(ctx->reorder, xid, buf->origptr);
+ ReorderBufferForget(ctx->reorder, xid, buf->origptr, commit_time);
return;
}
@@ -821,10 +822,11 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
for (i = 0; i < parsed->nsubxacts; i++)
{
ReorderBufferAbort(ctx->reorder, parsed->subxacts[i],
- buf->record->EndRecPtr);
+ buf->record->EndRecPtr, abort_time);
}
- ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr);
+ ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr,
+ abort_time);
}
/* update the decoding stats */
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 2bdab53e19..d11d2361c6 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -54,6 +54,7 @@
int max_logical_replication_workers = 4;
int max_sync_workers_per_subscription = 2;
+int max_apply_bgworkers_per_subscription = 2;
LogicalRepWorker *MyLogicalRepWorker = NULL;
@@ -73,6 +74,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg);
static void logicalrep_worker_onexit(int code, Datum arg);
static void logicalrep_worker_detach(void);
static void logicalrep_worker_cleanup(LogicalRepWorker *worker);
+static void logicalrep_worker_stop_internal(LogicalRepWorker *worker);
static bool on_commit_launcher_wakeup = false;
@@ -151,8 +153,10 @@ get_subscription_list(void)
*
* This is only needed for cleaning up the shared memory in case the worker
* fails to attach.
+ *
+ * Return false if the attach fails. Otherwise return true.
*/
-static void
+static bool
WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
uint16 generation,
BackgroundWorkerHandle *handle)
@@ -168,11 +172,11 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
- /* Worker either died or has started; no need to do anything. */
+ /* Worker either died or has started. Return false if died. */
if (!worker->in_use || worker->proc)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return worker->in_use;
}
LWLockRelease(LogicalRepWorkerLock);
@@ -187,7 +191,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
if (generation == worker->generation)
logicalrep_worker_cleanup(worker);
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -223,6 +227,13 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running)
{
LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+ /*
+ * We are only interested in the main apply worker or table sync worker
+ * here.
+ */
+ if (w->subworker)
+ continue;
+
if (w->in_use && w->subid == subid && w->relid == relid &&
(!only_running || w->proc))
{
@@ -259,11 +270,11 @@ logicalrep_workers_find(Oid subid, bool only_running)
}
/*
- * Start new apply background worker, if possible.
+ * Start new background worker, if possible.
*/
-void
+bool
logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
- Oid relid)
+ Oid relid, dsm_handle subworker_dsm)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
@@ -273,6 +284,10 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
LogicalRepWorker *worker = NULL;
int nsyncworkers;
TimestampTz now;
+ bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
+
+ /* Sanity check : we don't support table sync in subworker. */
+ Assert(!(is_subworker && OidIsValid(relid)));
ereport(DEBUG1,
(errmsg_internal("starting logical replication worker for subscription \"%s\"",
@@ -350,7 +365,7 @@ retry:
if (OidIsValid(relid) && nsyncworkers >= max_sync_workers_per_subscription)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -364,7 +379,7 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of logical replication worker slots"),
errhint("You might need to increase max_logical_replication_workers.")));
- return;
+ return false;
}
/* Prepare the worker slot. */
@@ -379,6 +394,7 @@ retry:
worker->relstate = SUBREL_STATE_UNKNOWN;
worker->relstate_lsn = InvalidXLogRecPtr;
worker->stream_fileset = NULL;
+ worker->subworker = is_subworker;
worker->last_lsn = InvalidXLogRecPtr;
TIMESTAMP_NOBEGIN(worker->last_send_time);
TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -396,19 +412,31 @@ retry:
BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
- snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
+ if (is_subworker)
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyBgworkerMain");
+ else
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
if (OidIsValid(relid))
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u sync %u", subid, relid);
+ else if (is_subworker)
+ snprintf(bgw.bgw_name, BGW_MAXLEN,
+ "logical replication apply background worker for subscription %u", subid);
else
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u", subid);
+
snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = Int32GetDatum(slot);
+ if (is_subworker)
+ memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle));
+
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
/* Failed to start worker, so clean up the worker slot. */
@@ -421,11 +449,11 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of background worker slots"),
errhint("You might need to increase max_worker_processes.")));
- return;
+ return false;
}
/* Now wait until it attaches. */
- WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
+ return WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
}
/*
@@ -436,18 +464,27 @@ void
logicalrep_worker_stop(Oid subid, Oid relid)
{
LogicalRepWorker *worker;
- uint16 generation;
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
worker = logicalrep_worker_find(subid, relid, false);
- /* No worker, nothing to do. */
- if (!worker)
- {
- LWLockRelease(LogicalRepWorkerLock);
- return;
- }
+ if (worker)
+ logicalrep_worker_stop_internal(worker);
+
+ LWLockRelease(LogicalRepWorkerLock);
+}
+
+/*
+ * Workhorse for logicalrep_worker_stop() and logicalrep_worker_detach(). Stop
+ * the worker and wait for it to die.
+ */
+static void
+logicalrep_worker_stop_internal(LogicalRepWorker *worker)
+{
+ uint16 generation;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
/*
* Remember which generation was our worker so we can check if what we see
@@ -485,10 +522,7 @@ logicalrep_worker_stop(Oid subid, Oid relid)
* different, meaning that a different worker has taken the slot.
*/
if (!worker->in_use || worker->generation != generation)
- {
- LWLockRelease(LogicalRepWorkerLock);
return;
- }
/* Worker has assigned proc, so it has started. */
if (worker->proc)
@@ -522,8 +556,6 @@ logicalrep_worker_stop(Oid subid, Oid relid)
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
}
-
- LWLockRelease(LogicalRepWorkerLock);
}
/*
@@ -599,6 +631,29 @@ logicalrep_worker_attach(int slot)
static void
logicalrep_worker_detach(void)
{
+ /*
+ * If we are the main apply worker, stop all the apply background workers
+ * we started before.
+ */
+ if (!MyLogicalRepWorker->subworker)
+ {
+ List *workers;
+ ListCell *lc;
+
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+
+ workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true);
+ foreach(lc, workers)
+ {
+ LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc);
+
+ if (w->subworker)
+ logicalrep_worker_stop_internal(w);
+ }
+
+ LWLockRelease(LogicalRepWorkerLock);
+ }
+
/* Block concurrent access. */
LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
@@ -621,6 +676,7 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker)
worker->userid = InvalidOid;
worker->subid = InvalidOid;
worker->relid = InvalidOid;
+ worker->subworker = false;
}
/*
@@ -679,6 +735,30 @@ logicalrep_sync_worker_count(Oid subid)
return res;
}
+/*
+ * Count the number of registered (not necessarily running) apply background
+ * workers for a subscription.
+ */
+int
+logicalrep_apply_bgworker_count(Oid subid)
+{
+ int i;
+ int res = 0;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
+
+ /* Search for attached worker for a given subscription id. */
+ for (i = 0; i < max_logical_replication_workers; i++)
+ {
+ LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+
+ if (w->subid == subid && w->subworker)
+ res++;
+ }
+
+ return res;
+}
+
/*
* ApplyLauncherShmemSize
* Compute space needed for replication launcher shared memory
@@ -868,7 +948,7 @@ ApplyLauncherMain(Datum main_arg)
wait_time = wal_retrieve_retry_interval;
logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
- sub->owner, InvalidOid);
+ sub->owner, InvalidOid, DSM_HANDLE_INVALID);
}
}
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index 21937ab2d3..50c567fb6e 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1063,12 +1063,21 @@ ReplicationOriginExitCleanup(int code, Datum arg)
* array doesn't have to be searched when calling
* replorigin_session_advance().
*
- * Obviously only one such cached origin can exist per process and the current
+ * Normally only one such cached origin can exist per process and the current
* cached value can only be set again after the previous value is torn down
* with replorigin_session_reset().
+ *
+ * However, if the function parameter 'must_acquire' is false, we allow the
+ * process to use the same slot already acquired by another process. It's safe
+ * because 1) The only caller (apply background workers) will maintain the
+ * commit order by allowing only one process to commit at a time, so no two
+ * workers will be operating on the same origin at the same time (see comments
+ * in logical/worker.c). 2) Even though we try to advance the session's origin
+ * concurrently, it's safe to do so as we change/advance the session_origin
+ * LSNs under replicate_state LWLock.
*/
void
-replorigin_session_setup(RepOriginId node)
+replorigin_session_setup(RepOriginId node, bool must_acquire)
{
static bool registered_cleanup;
int i;
@@ -1110,7 +1119,7 @@ replorigin_session_setup(RepOriginId node)
if (curstate->roident != node)
continue;
- else if (curstate->acquired_by != 0)
+ else if (curstate->acquired_by != 0 && must_acquire)
{
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
@@ -1141,7 +1150,14 @@ replorigin_session_setup(RepOriginId node)
Assert(session_replication_state->roident != InvalidRepOriginId);
- session_replication_state->acquired_by = MyProcPid;
+ if (must_acquire)
+ session_replication_state->acquired_by = MyProcPid;
+ else if (session_replication_state->acquired_by == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+ errmsg("apply background worker could not find replication state slot for replication origin with OID %u",
+ node),
+ errdetail("There is no replication state slot set by its main apply worker.")));
LWLockRelease(ReplicationOriginLock);
@@ -1321,7 +1337,7 @@ pg_replication_origin_session_setup(PG_FUNCTION_ARGS)
name = text_to_cstring((text *) DatumGetPointer(PG_GETARG_DATUM(0)));
origin = replorigin_by_name(name, false);
- replorigin_session_setup(origin);
+ replorigin_session_setup(origin, true);
replorigin_session_origin = origin;
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index ff8513e2d2..affd08cfa4 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -1163,31 +1163,56 @@ logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data)
/*
* Write STREAM ABORT to the output stream. Note that xid and subxid will be
* same for the top-level transaction abort.
+ *
+ * If write_abort_lsn is true, send the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
void
logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid)
+ ReorderBufferTXN *txn, XLogRecPtr abort_lsn,
+ bool write_abort_lsn)
{
pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT);
- Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid));
+ Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(txn->xid));
/* transaction ID */
pq_sendint32(out, xid);
- pq_sendint32(out, subxid);
+ pq_sendint32(out, txn->xid);
+
+ if (write_abort_lsn)
+ {
+ pq_sendint64(out, abort_lsn);
+ pq_sendint64(out, txn->xact_time.abort_time);
+ }
}
/*
* Read STREAM ABORT from the output stream.
+ *
+ * If read_abort_lsn is true, try to read the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
void
-logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid)
+logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn)
{
- Assert(xid && subxid);
+ Assert(abort_data);
- *xid = pq_getmsgint(in, 4);
- *subxid = pq_getmsgint(in, 4);
+ abort_data->xid = pq_getmsgint(in, 4);
+ abort_data->subxid = pq_getmsgint(in, 4);
+
+ if (read_abort_lsn)
+ {
+ abort_data->abort_lsn = pq_getmsgint64(in);
+ abort_data->abort_time = pq_getmsgint64(in);
+ }
+ else
+ {
+ abort_data->abort_lsn = InvalidXLogRecPtr;
+ abort_data->abort_time = 0;
+ }
}
/*
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 88a37fde72..8989328046 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -2826,7 +2826,8 @@ ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
* disk.
*/
void
-ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2837,6 +2838,8 @@ ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
{
@@ -2911,7 +2914,8 @@ ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid)
* to this xid might re-create the transaction incompletely.
*/
void
-ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2922,6 +2926,8 @@ ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
rb->stream_abort(rb, txn, lsn);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 670c6fcada..8ffba7e2e5 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -568,7 +568,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
MySubscription->oid,
MySubscription->name,
MyLogicalRepWorker->userid,
- rstate->relid);
+ rstate->relid,
+ DSM_HANDLE_INVALID);
hentry->last_start_time = now;
}
}
@@ -589,6 +590,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
void
process_syncing_tables(XLogRecPtr current_lsn)
{
+ if (MyLogicalRepWorker->subworker)
+ return;
+
if (am_tablesync_worker())
process_syncing_tables_for_sync(current_lsn);
else
@@ -1273,7 +1277,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
* time this tablesync was launched.
*/
originid = replorigin_by_name(originname, false);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
*origin_startpos = replorigin_session_get_progress(false);
@@ -1384,7 +1388,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
true /* go backward */ , true /* WAL log */ );
UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
}
else
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 38e3b1c1b3..bbd0304a8d 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -22,8 +22,28 @@
* STREAMED TRANSACTIONS
* ---------------------
* Streamed transactions (large transactions exceeding a memory limit on the
- * upstream) are not applied immediately, but instead, the data is written
- * to temporary files and then applied at once when the final commit arrives.
+ * upstream) are applied using one of two approaches.
+ *
+ * 1) Separate background workers
+ *
+ * If streaming = parallel, We assign a new apply background worker (if
+ * available) as soon as the xact's first stream is received. The main apply
+ * worker will send changes to this new worker via shared memory. We keep this
+ * worker assigned till the transaction commit is received and also wait for
+ * the worker to finish at commit. This preserves commit ordering and avoids
+ * file I/O in most cases. We still need to spill to a file if there is no
+ * worker available. It is important to maintain commit order to avoid failures
+ * due to (a) transaction dependencies, say if we insert a row in the first
+ * transaction and update it in the second transaction then allowing to apply
+ * both in parallel can lead to failure in the update. (b) deadlocks, allowing
+ * transactions that update the same set of rows/tables in opposite order to be
+ * applied in parallel can lead to deadlocks.
+ *
+ * 2) Write to temporary files and apply when the final commit arrives
+ *
+ * If no worker is available to handle streamed transaction, the data is
+ * written to temporary files and then applied at once when the final commit
+ * arrives.
*
* Unlike the regular (non-streamed) case, handling streamed transactions has
* to handle aborts of both the toplevel transaction and subtransactions. This
@@ -219,20 +239,8 @@ typedef struct ApplyExecutionData
PartitionTupleRouting *proute; /* partition routing info */
} ApplyExecutionData;
-/* Struct for saving and restoring apply errcontext information */
-typedef struct ApplyErrorCallbackArg
-{
- LogicalRepMsgType command; /* 0 if invalid */
- LogicalRepRelMapEntry *rel;
-
- /* Remote node information */
- int remote_attnum; /* -1 if invalid */
- TransactionId remote_xid;
- XLogRecPtr finish_lsn;
- char *origin_name;
-} ApplyErrorCallbackArg;
-
-static ApplyErrorCallbackArg apply_error_callback_arg =
+/* errcontext tracker */
+ApplyErrorCallbackArg apply_error_callback_arg =
{
.command = 0,
.rel = NULL,
@@ -242,7 +250,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg =
.origin_name = NULL,
};
-static MemoryContext ApplyMessageContext = NULL;
+MemoryContext ApplyMessageContext = NULL;
MemoryContext ApplyContext = NULL;
/* per stream context for streaming transactions */
@@ -251,27 +259,38 @@ static MemoryContext LogicalStreamingContext = NULL;
WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
Subscription *MySubscription = NULL;
-static bool MySubscriptionValid = false;
+bool MySubscriptionValid = false;
bool in_remote_transaction = false;
static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
/* fields valid only when processing streamed transaction */
-static bool in_streamed_transaction = false;
+bool in_streamed_transaction = false;
+
+TransactionId stream_xid = InvalidTransactionId;
+static ApplyBgworkerState *stream_apply_worker = NULL;
-static TransactionId stream_xid = InvalidTransactionId;
+/* Check if we are applying the transaction in an apply background worker */
+#define apply_bgworker_active() (in_streamed_transaction && stream_apply_worker != NULL)
+
+/*
+ * The number of changes during one streaming block (only for apply background
+ * workers)
+ */
+static uint32 nchanges = 0;
/*
* We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
* the subscription if the remote transaction's finish LSN matches the subskiplsn.
* Once we start skipping changes, we don't stop it until we skip all changes of
* the transaction even if pg_subscription is updated and MySubscription->skiplsn
- * gets changed or reset during that. Also, in streaming transaction cases, we
- * don't skip receiving and spooling the changes since we decide whether or not
+ * gets changed or reset during that. Also, in streaming transaction cases (streaming = on),
+ * we don't skip receiving and spooling the changes since we decide whether or not
* to skip applying the changes when starting to apply changes. The subskiplsn is
* cleared after successfully skipping the transaction or applying non-empty
* transaction. The latter prevents the mistakenly specified subskiplsn from
- * being left.
+ * being left. Note that we cannot skip the streaming transaction in parallel
+ * mode, because we cannot get the finish LSN before applying the changes.
*/
static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr;
#define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn)))
@@ -324,9 +343,6 @@ static void maybe_reread_subscription(void);
static void DisableSubscriptionAndExit(void);
-/* prototype needed because of stream_commit */
-static void apply_dispatch(StringInfo s);
-
static void apply_handle_commit_internal(LogicalRepCommitData *commit_data);
static void apply_handle_insert_internal(ApplyExecutionData *edata,
ResultRelInfo *relinfo,
@@ -359,7 +375,6 @@ static void stop_skipping_changes(void);
static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
/* Functions for apply error callback */
-static void apply_error_callback(void *arg);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
@@ -426,40 +441,85 @@ end_replication_step(void)
}
/*
- * Handle streamed transactions.
+ * Handle streamed transactions for both the main apply worker and the apply
+ * background workers.
+ *
+ * In streaming case (receiving a block of streamed transaction), for
+ * SUBSTREAM_ON mode, simply redirect it to a file for the proper toplevel
+ * transaction, and for SUBSTREAM_PARALLEL mode, send the changes to apply
+ * background workers (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes
+ * will also be applied in main apply worker).
*
- * If in streaming mode (receiving a block of streamed transaction), we
- * simply redirect it to a file for the proper toplevel transaction.
+ * For non-streamed transactions, returns false;
+ * For streamed transactions, returns true if in main apply worker, false
+ * otherwise.
*
- * Returns true for streamed transactions, false otherwise (regular mode).
+ * Exception: When parallel mode is applying streamed transaction in the main
+ * apply worker, (e.g. when addressing LOGICAL_REP_MSG_RELATION or
+ * LOGICAL_REP_MSG_TYPE changes), then return false.
*/
static bool
handle_streamed_transaction(LogicalRepMsgType action, StringInfo s)
{
- TransactionId xid;
+ TransactionId current_xid = InvalidTransactionId;
- /* not in streaming mode */
- if (!in_streamed_transaction)
+ /* Not in streaming mode */
+ if (!(in_streamed_transaction || am_apply_bgworker()))
return false;
- Assert(stream_fd != NULL);
Assert(TransactionIdIsValid(stream_xid));
/*
* We should have received XID of the subxact as the first part of the
* message, so extract it.
*/
- xid = pq_getmsgint(s, 4);
+ current_xid = pq_getmsgint(s, 4);
- if (!TransactionIdIsValid(xid))
+ if (!TransactionIdIsValid(current_xid))
ereport(ERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("invalid transaction ID in streamed replication transaction")));
- /* Add the new subxact to the array (unless already there). */
- subxact_info_add(xid);
+ if (am_apply_bgworker())
+ {
+ /* Define a savepoint for a subxact if needed. */
+ apply_bgworker_subxact_info_add(current_xid);
+
+ return false;
+ }
+
+ if (apply_bgworker_active())
+ {
+ /*
+ * This is the main apply worker, but there is an apply background
+ * worker, so apply the changes of this transaction in that background
+ * worker. Pass the data to the worker.
+ */
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
+ nchanges += 1;
+
+ /*
+ * XXX The publisher side doesn't always send relation/type update
+ * messages after the streaming transaction, so also update the
+ * relation/type in main apply worker here. See function
+ * cleanup_rel_sync_cache.
+ */
+ if (action == LOGICAL_REP_MSG_RELATION ||
+ action == LOGICAL_REP_MSG_TYPE)
+ return false;
+
+ return true;
+ }
+
+ /*
+ * This is the main apply worker, but there is no apply background worker,
+ * so write to temporary files and apply when the final commit arrives.
+ *
+ * Add the new subxact to the array (unless already there).
+ */
+ subxact_info_add(current_xid);
- /* write the change to the current file */
+ /* Write the change to the current file */
stream_write_change(action, s);
return true;
@@ -844,6 +904,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -898,7 +961,9 @@ apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data)
* BeginTransactionBlock is necessary to balance the EndTransactionBlock
* called within the PrepareTransactionBlock below.
*/
- BeginTransactionBlock();
+ if (!IsTransactionBlock())
+ BeginTransactionBlock();
+
CommitTransactionCommand(); /* Completes the preceding Begin command. */
/*
@@ -950,6 +1015,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1064,10 +1132,6 @@ apply_handle_rollback_prepared(StringInfo s)
/*
* Handle STREAM PREPARE.
- *
- * Logic is in two parts:
- * 1. Replay all the spooled operations
- * 2. Mark the transaction as prepared
*/
static void
apply_handle_stream_prepare(StringInfo s)
@@ -1088,24 +1152,76 @@ apply_handle_stream_prepare(StringInfo s)
logicalrep_read_stream_prepare(s, &prepare_data);
set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn);
- elog(DEBUG1, "received prepare for streamed transaction %u", prepare_data.xid);
+ if (am_apply_bgworker())
+ {
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
- /* Replay all the spooled operations. */
- apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+ CommitTransactionCommand();
- /* Mark the transaction as prepared. */
- apply_handle_prepare_internal(&prepare_data);
+ pgstat_report_stat(false);
- CommitTransactionCommand();
+ list_free(subxactlist);
+ subxactlist = NIL;
- pgstat_report_stat(false);
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(prepare_data.xid);
- store_flush_position(prepare_data.end_lsn);
+ elog(DEBUG1, "received prepare for streamed transaction %u",
+ prepare_data.xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+
+ pgstat_report_stat(false);
+ store_flush_position(prepare_data.end_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
+
+ CommitTransactionCommand();
+
+ pgstat_report_stat(false);
+
+ store_flush_position(prepare_data.end_lsn);
+
+ in_remote_transaction = false;
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ }
+ }
in_remote_transaction = false;
+ stream_apply_worker = NULL;
- /* unlink the files with serialized changes and subxact info. */
- stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1155,15 +1271,6 @@ apply_handle_stream_start(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("duplicate STREAM START message")));
- /*
- * Start a transaction on stream start, this transaction will be committed
- * on the stream stop unless it is a tablesync worker in which case it
- * will be committed after processing all the messages. We need the
- * transaction for handling the buffile, used for serializing the
- * streaming data and subxact info.
- */
- begin_replication_step();
-
/* notify handle methods we're processing a remote transaction */
in_streamed_transaction = true;
@@ -1177,36 +1284,93 @@ apply_handle_stream_start(StringInfo s)
set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr);
- /*
- * Initialize the worker's stream_fileset if we haven't yet. This will be
- * used for the entire duration of the worker so create it in a permanent
- * context. We create this on the very first streaming message from any
- * transaction and then use it for this and other streaming transactions.
- * Now, we could create a fileset at the start of the worker as well but
- * then we won't be sure that it will ever be used.
- */
- if (MyLogicalRepWorker->stream_fileset == NULL)
+ if (am_apply_bgworker())
{
- MemoryContext oldctx;
-
- oldctx = MemoryContextSwitchTo(ApplyContext);
+ /*
+ * Make sure the handle apply_dispatch methods are aware we're in a
+ * remote transaction.
+ */
+ in_remote_transaction = true;
- MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
- FileSetInit(MyLogicalRepWorker->stream_fileset);
+ /* Begin the transaction. */
+ AcceptInvalidationMessages();
+ maybe_reread_subscription();
- MemoryContextSwitchTo(oldctx);
+ StartTransactionCommand();
+ BeginTransactionBlock();
+ CommitTransactionCommand();
}
+ else
+ {
+ /*
+ * This is the main apply worker. Check if there is any free apply
+ * background worker we can use to process this transaction.
+ */
+ if (first_segment)
+ stream_apply_worker = apply_bgworker_start(stream_xid);
+ else
+ stream_apply_worker = apply_bgworker_find(stream_xid);
- /* open the spool file for this transaction */
- stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+ if (stream_apply_worker)
+ {
+ /*
+ * If we have found a free worker or if we are already applying this
+ * transaction in an apply background worker, then we pass the data to
+ * that worker.
+ */
+ if (first_segment)
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
- /* if this is not the first segment, open existing subxact file */
- if (!first_segment)
- subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+ nchanges = 0;
+ elog(DEBUG1, "starting streaming of xid %u", stream_xid);
+ }
+ else
+ {
+ /*
+ * Since no apply background worker is available for the first
+ * stream start, serialize all the changes of the transaction.
+ *
+ * Start a transaction on stream start, this transaction will be
+ * committed on the stream stop unless it is a tablesync worker in
+ * which case it will be committed after processing all the
+ * messages. We need the transaction for handling the buffile,
+ * used for serializing the streaming data and subxact info.
+ */
+ begin_replication_step();
- pgstat_report_activity(STATE_RUNNING, NULL);
+ /*
+ * Initialize the worker's stream_fileset if we haven't yet. This will
+ * be used for the entire duration of the worker so create it in a
+ * permanent context. We create this on the very first streaming
+ * message from any transaction and then use it for this and other
+ * streaming transactions. Now, we could create a fileset at the start
+ * of the worker as well but then we won't be sure that it will ever
+ * be used.
+ */
+ if (MyLogicalRepWorker->stream_fileset == NULL)
+ {
+ MemoryContext oldctx;
- end_replication_step();
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
+ FileSetInit(MyLogicalRepWorker->stream_fileset);
+
+ MemoryContextSwitchTo(oldctx);
+ }
+
+ /* Open the spool file for this transaction. */
+ stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+
+ /* If this is not the first segment, open existing subxact file. */
+ if (!first_segment)
+ subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+
+ end_replication_step();
+ }
+ }
+
+ pgstat_report_activity(STATE_RUNNING, NULL);
}
/*
@@ -1220,53 +1384,52 @@ apply_handle_stream_stop(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("STREAM STOP message without STREAM START")));
- /*
- * Close the file with serialized changes, and serialize information about
- * subxacts for the toplevel transaction.
- */
- subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
- stream_close_file();
+ if (apply_bgworker_active())
+ {
+ char action = LOGICAL_REP_MSG_STREAM_STOP;
- /* We must be in a valid transaction state */
- Assert(IsTransactionState());
+ apply_bgworker_send_data(stream_apply_worker, 1, &action);
- /* Commit the per-stream transaction */
- CommitTransactionCommand();
+ elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
+ }
+ else
+ {
+ /*
+ * Close the file with serialized changes, and serialize information
+ * about subxacts for the toplevel transaction.
+ */
+ subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
+ stream_close_file();
- in_streamed_transaction = false;
+ /* We must be in a valid transaction state */
+ Assert(IsTransactionState());
- /* Reset per-stream context */
- MemoryContextReset(LogicalStreamingContext);
+ /* Commit the per-stream transaction */
+ CommitTransactionCommand();
+
+ /* Reset per-stream context */
+ MemoryContextReset(LogicalStreamingContext);
+ }
+
+ in_streamed_transaction = false;
+ stream_apply_worker = NULL;
pgstat_report_activity(STATE_IDLE, NULL);
reset_apply_error_context_info();
}
/*
- * Handle STREAM abort message.
+ * Handle STREAM ABORT message when the transaction was spilled to disk.
*/
static void
-apply_handle_stream_abort(StringInfo s)
+serialize_stream_abort(TransactionId xid, TransactionId subxid)
{
- TransactionId xid;
- TransactionId subxid;
-
- if (in_streamed_transaction)
- ereport(ERROR,
- (errcode(ERRCODE_PROTOCOL_VIOLATION),
- errmsg_internal("STREAM ABORT message without STREAM STOP")));
-
- logicalrep_read_stream_abort(s, &xid, &subxid);
-
/*
* If the two XIDs are the same, it's in fact abort of toplevel xact, so
* just delete the files with serialized info.
*/
if (xid == subxid)
- {
- set_apply_error_context_xact(xid, InvalidXLogRecPtr);
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
- }
else
{
/*
@@ -1290,8 +1453,6 @@ apply_handle_stream_abort(StringInfo s)
bool found = false;
char path[MAXPGPATH];
- set_apply_error_context_xact(subxid, InvalidXLogRecPtr);
-
subidx = -1;
begin_replication_step();
subxact_info_read(MyLogicalRepWorker->subid, xid);
@@ -1316,7 +1477,6 @@ apply_handle_stream_abort(StringInfo s)
cleanup_subxact_info();
end_replication_step();
CommitTransactionCommand();
- reset_apply_error_context_info();
return;
}
@@ -1339,6 +1499,142 @@ apply_handle_stream_abort(StringInfo s)
end_replication_step();
CommitTransactionCommand();
}
+}
+
+/*
+ * Handle STREAM ABORT message.
+ */
+static void
+apply_handle_stream_abort(StringInfo s)
+{
+ TransactionId xid;
+ TransactionId subxid;
+ LogicalRepStreamAbortData abort_data;
+ bool read_abort_lsn = false;
+
+ if (in_streamed_transaction)
+ ereport(ERROR,
+ (errcode(ERRCODE_PROTOCOL_VIOLATION),
+ errmsg_internal("STREAM ABORT message without STREAM STOP")));
+
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelState->server_version >= 160000;
+
+ logicalrep_read_stream_abort(s, &abort_data, read_abort_lsn);
+
+ xid = abort_data.xid;
+ subxid = abort_data.subxid;
+
+ set_apply_error_context_xact(subxid, abort_data.abort_lsn);
+
+ if (am_apply_bgworker())
+ {
+ elog(DEBUG1, "[Apply BGW #%u] aborting current transaction xid=%u, subxid=%u",
+ MyParallelState->n, GetCurrentTransactionIdIfAny(),
+ GetCurrentSubTransactionId());
+
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ if (read_abort_lsn)
+ {
+ replorigin_session_origin_lsn = abort_data.abort_lsn;
+ replorigin_session_origin_timestamp = abort_data.abort_time;
+ }
+
+ /*
+ * If the two XIDs are the same, it's in fact abort of toplevel xact,
+ * so just free the subxactlist.
+ */
+ if (subxid == xid)
+ {
+ AbortCurrentTransaction();
+
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
+
+ in_remote_transaction = false;
+ pgstat_report_activity(STATE_IDLE, NULL);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /*
+ * OK, so it's a subxact. Rollback to the savepoint.
+ *
+ * We also need to read the subxactlist, determine the offset
+ * tracked for the subxact, and truncate the list.
+ */
+ int i;
+ bool found = false;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", subxid);
+
+ elog(DEBUG1, "[Apply BGW #%u] rolling back to savepoint %s",
+ MyParallelState->n, spname);
+
+ for (i = list_length(subxactlist) - 1; i >= 0; i--)
+ {
+ xid = (TransactionId) list_nth_int(subxactlist, i);
+ if (xid == subxid)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if (found)
+ {
+ RollbackToSavepoint(spname);
+ CommitTransactionCommand();
+ subxactlist = list_truncate(subxactlist, i + 1);
+ }
+
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ }
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM ABORT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ if (subxid == xid)
+ {
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+ }
+ }
+ else
+ {
+ /*
+ * We are in main apply worker and the transaction has been
+ * serialized to file.
+ */
+ serialize_stream_abort(xid, subxid);
+ }
+ }
reset_apply_error_context_info();
}
@@ -1468,8 +1764,8 @@ apply_spooled_messages(TransactionId xid, XLogRecPtr lsn)
static void
apply_handle_stream_commit(StringInfo s)
{
- TransactionId xid;
LogicalRepCommitData commit_data;
+ TransactionId xid;
if (in_streamed_transaction)
ereport(ERROR,
@@ -1479,14 +1775,81 @@ apply_handle_stream_commit(StringInfo s)
xid = logicalrep_read_stream_commit(s, &commit_data);
set_apply_error_context_xact(xid, commit_data.commit_lsn);
- elog(DEBUG1, "received commit for streamed transaction %u", xid);
+ if (am_apply_bgworker())
+ {
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ replorigin_session_origin_lsn = commit_data.end_lsn;
+ replorigin_session_origin_timestamp = commit_data.committime;
+
+ CommitTransactionCommand();
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
- apply_spooled_messages(xid, commit_data.commit_lsn);
+ in_remote_transaction = false;
- apply_handle_commit_internal(&commit_data);
+ pgstat_report_stat(false);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ elog(DEBUG1, "received commit for streamed transaction %u", xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM COMMIT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
- /* unlink the files with serialized changes and subxact info */
- stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ pgstat_report_stat(false);
+ store_flush_position(commit_data.end_lsn);
+ stop_skipping_changes();
+
+ apply_bgworker_free(wstate);
+
+ /*
+ * The transaction is either non-empty or skipped, so we clear the
+ * subskiplsn.
+ */
+ clear_subscription_skip_lsn(commit_data.commit_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(xid, commit_data.commit_lsn);
+
+ apply_handle_commit_internal(&commit_data);
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ }
+ }
+
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -2467,7 +2830,7 @@ apply_handle_truncate(StringInfo s)
/*
* Logical replication protocol message dispatcher.
*/
-static void
+void
apply_dispatch(StringInfo s)
{
LogicalRepMsgType action = pq_getmsgbyte(s);
@@ -2636,6 +2999,10 @@ store_flush_position(XLogRecPtr remote_lsn)
{
FlushPosition *flushpos;
+ /* We only need to collect the LSN in main apply worker */
+ if (am_apply_bgworker())
+ return;
+
/* Need to do this in permanent context */
MemoryContextSwitchTo(ApplyContext);
@@ -2650,7 +3017,7 @@ store_flush_position(XLogRecPtr remote_lsn)
/* Update statistics of the worker. */
-static void
+void
UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply)
{
MyLogicalRepWorker->last_lsn = last_lsn;
@@ -2812,6 +3179,9 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
AcceptInvalidationMessages();
maybe_reread_subscription();
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any table synchronization changes. */
process_syncing_tables(last_received);
}
@@ -3113,7 +3483,7 @@ maybe_reread_subscription(void)
/*
* Callback from subscription syscache invalidation.
*/
-static void
+void
subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue)
{
MySubscriptionValid = false;
@@ -3709,7 +4079,7 @@ ApplyWorkerMain(Datum main_arg)
originid = replorigin_by_name(originname, true);
if (!OidIsValid(originid))
originid = replorigin_create(originname);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
origin_startpos = replorigin_session_get_progress(false);
CommitTransactionCommand();
@@ -3750,13 +4120,14 @@ ApplyWorkerMain(Datum main_arg)
server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
options.proto.logical.proto_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
LOGICALREP_PROTO_VERSION_NUM;
options.proto.logical.publication_names = MySubscription->publications;
options.proto.logical.binary = MySubscription->binary;
- options.proto.logical.streaming = MySubscription->stream;
+ options.proto.logical.streaming = (MySubscription->stream != SUBSTREAM_OFF);
options.proto.logical.twophase = false;
if (!am_tablesync_worker())
@@ -3914,7 +4285,8 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
XLogRecPtr myskiplsn = MySubscription->skiplsn;
bool started_tx = false;
- if (likely(XLogRecPtrIsInvalid(myskiplsn)))
+ if (likely(XLogRecPtrIsInvalid(myskiplsn)) ||
+ am_apply_bgworker())
return;
if (!IsTransactionState())
@@ -3986,7 +4358,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
}
/* Error callback to give more context info about the change being applied */
-static void
+void
apply_error_callback(void *arg)
{
ApplyErrorCallbackArg *errarg = &apply_error_callback_arg;
@@ -4014,23 +4386,47 @@ apply_error_callback(void *arg)
errarg->remote_xid,
LSN_FORMAT_ARGS(errarg->finish_lsn));
}
- else if (errarg->remote_attnum < 0)
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
else
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->rel->remoterel.attnames[errarg->remote_attnum],
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
+ {
+ if (errarg->remote_attnum < 0)
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ else
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ }
}
/* Set transaction information of apply error callback */
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 2cbca4a087..1aaca04982 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -1820,6 +1820,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
XLogRecPtr abort_lsn)
{
ReorderBufferTXN *toptxn;
+ bool write_abort_lsn = false;
+ PGOutputData *data = (PGOutputData *) ctx->output_plugin_private;
/*
* The abort should happen outside streaming block, even for streamed
@@ -1832,8 +1834,13 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
Assert(rbtxn_is_streamed(toptxn));
+ /* We only send abort_lsn and abort_time if the subscriber needs them. */
+ if (data->protocol_version >= LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM)
+ write_abort_lsn = true;
+
OutputPluginPrepareWrite(ctx, true);
- logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid);
+ logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn, abort_lsn,
+ write_abort_lsn);
OutputPluginWrite(ctx, true);
cleanup_rel_sync_cache(toptxn->xid, false);
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index 87c15b9c6f..ba781e6f08 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -388,6 +388,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT:
event_name = "HashGrowBucketsReinsert";
break;
+ case WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE:
+ event_name = "LogicalApplyWorkerStateChange";
+ break;
case WAIT_EVENT_LOGICAL_SYNC_DATA:
event_name = "LogicalSyncData";
break;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d43..4284bcbcd1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3220,6 +3220,18 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_apply_bgworkers_per_subscription",
+ PGC_SIGHUP,
+ REPLICATION_SUBSCRIBERS,
+ gettext_noop("Maximum number of apply background workers per subscription."),
+ NULL,
+ },
+ &max_apply_bgworkers_per_subscription,
+ 2, 0, MAX_BACKENDS,
+ NULL, NULL, NULL
+ },
+
{
{"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Sets the amount of time to wait before forcing "
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..ad18710af4 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,7 @@
#max_logical_replication_workers = 4 # taken from max_worker_processes
# (change requires restart)
#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers
+#max_apply_bgworkers_per_subscription = 2 # taken from max_logical_replication_workers
#------------------------------------------------------------------------------
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f317f0a681..24927641b9 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4449,7 +4449,7 @@ getSubscriptions(Archive *fout)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query, " s.substream,\n");
else
- appendPQExpBufferStr(query, " false AS substream,\n");
+ appendPQExpBufferStr(query, " 'f' AS substream,\n");
if (fout->remoteVersion >= 150000)
appendPQExpBufferStr(query,
@@ -4579,8 +4579,10 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo)
if (strcmp(subinfo->subbinary, "t") == 0)
appendPQExpBufferStr(query, ", binary = true");
- if (strcmp(subinfo->substream, "f") != 0)
+ if (strcmp(subinfo->substream, "t") == 0)
appendPQExpBufferStr(query, ", streaming = on");
+ else if (strcmp(subinfo->substream, "p") == 0)
+ appendPQExpBufferStr(query, ", streaming = parallel");
if (strcmp(subinfo->subtwophasestate, two_phase_disabled) != 0)
appendPQExpBufferStr(query, ", two_phase = on");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d1260f590c..d54540f5f5 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -68,7 +68,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subbinary; /* True if the subscription wants the
* publisher to send data in binary */
- bool substream; /* Stream in-progress transactions. */
+ char substream; /* Stream in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char subtwophasestate; /* Stream two-phase transactions */
@@ -109,7 +110,8 @@ typedef struct Subscription
bool enabled; /* Indicates if the subscription is enabled */
bool binary; /* Indicates if the subscription wants data in
* binary format */
- bool stream; /* Allow streaming in-progress transactions. */
+ char stream; /* Allow streaming in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char twophasestate; /* Allow streaming two-phase transactions */
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
@@ -120,6 +122,21 @@ typedef struct Subscription
List *publications; /* List of publication names to subscribe to */
} Subscription;
+/* Disallow streaming in-progress transactions */
+#define SUBSTREAM_OFF 'f'
+
+/*
+ * Streaming in-progress transactions are written to a temporary file and
+ * applied only after the transaction is committed on upstream.
+ */
+#define SUBSTREAM_ON 't'
+
+/*
+ * Streaming in-progress transactions are applied immediately via a background
+ * worker
+ */
+#define SUBSTREAM_PARALLEL 'p'
+
extern Subscription *GetSubscription(Oid subid, bool missing_ok);
extern void FreeSubscription(Subscription *sub);
extern void DisableSubscription(Oid subid);
diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h
index f1e2821e25..ac8ef94381 100644
--- a/src/include/replication/logicallauncher.h
+++ b/src/include/replication/logicallauncher.h
@@ -14,6 +14,7 @@
extern PGDLLIMPORT int max_logical_replication_workers;
extern PGDLLIMPORT int max_sync_workers_per_subscription;
+extern PGDLLIMPORT int max_apply_bgworkers_per_subscription;
extern void ApplyLauncherRegister(void);
extern void ApplyLauncherMain(Datum main_arg);
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index a771ab8ff3..0f74a3392b 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -32,12 +32,17 @@
*
* LOGICALREP_PROTO_TWOPHASE_VERSION_NUM is the minimum protocol version with
* support for two-phase commit decoding (at prepare time). Introduced in PG15.
+ *
+ * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version
+ * with support for streaming large transactions in apply background worker.
+ * Introduced in PG16.
*/
#define LOGICALREP_PROTO_MIN_VERSION_NUM 1
#define LOGICALREP_PROTO_VERSION_NUM 1
#define LOGICALREP_PROTO_STREAM_VERSION_NUM 2
#define LOGICALREP_PROTO_TWOPHASE_VERSION_NUM 3
-#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_TWOPHASE_VERSION_NUM
+#define LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM 4
+#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM
/*
* Logical message types
@@ -175,6 +180,17 @@ typedef struct LogicalRepRollbackPreparedTxnData
char gid[GIDSIZE];
} LogicalRepRollbackPreparedTxnData;
+/*
+ * Transaction protocol information for stream abort.
+ */
+typedef struct LogicalRepStreamAbortData
+{
+ TransactionId xid;
+ TransactionId subxid;
+ XLogRecPtr abort_lsn;
+ TimestampTz abort_time;
+} LogicalRepStreamAbortData;
+
extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn);
extern void logicalrep_read_begin(StringInfo in,
LogicalRepBeginData *begin_data);
@@ -246,9 +262,12 @@ extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn
extern TransactionId logicalrep_read_stream_commit(StringInfo out,
LogicalRepCommitData *commit_data);
extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid);
-extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid);
+ ReorderBufferTXN *txn,
+ XLogRecPtr abort_lsn,
+ bool write_abort_lsn);
+extern void logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool include_abort_lsn);
extern char *logicalrep_message_type(LogicalRepMsgType action);
#endif /* LOGICAL_PROTO_H */
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index cd1b6e8afc..6a1af7f13c 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -13,6 +13,7 @@
#define LOGICALWORKER_H
extern void ApplyWorkerMain(Datum main_arg);
+extern void ApplyBgworkerMain(Datum main_arg);
extern bool IsLogicalWorker(void);
diff --git a/src/include/replication/origin.h b/src/include/replication/origin.h
index 2d1b5e5c28..c7389b40a7 100644
--- a/src/include/replication/origin.h
+++ b/src/include/replication/origin.h
@@ -53,7 +53,7 @@ extern XLogRecPtr replorigin_get_progress(RepOriginId node, bool flush);
extern void replorigin_session_advance(XLogRecPtr remote_commit,
XLogRecPtr local_commit);
-extern void replorigin_session_setup(RepOriginId node);
+extern void replorigin_session_setup(RepOriginId node, bool must_acquire);
extern void replorigin_session_reset(void);
extern XLogRecPtr replorigin_session_get_progress(bool flush);
diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index d109d0baed..d2a80d79e5 100644
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -301,6 +301,7 @@ typedef struct ReorderBufferTXN
{
TimestampTz commit_time;
TimestampTz prepare_time;
+ TimestampTz abort_time;
} xact_time;
/*
@@ -647,9 +648,11 @@ extern void ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
extern void ReorderBufferAssignChild(ReorderBuffer *, TransactionId, TransactionId, XLogRecPtr commit_lsn);
extern void ReorderBufferCommitChild(ReorderBuffer *, TransactionId, TransactionId,
XLogRecPtr commit_lsn, XLogRecPtr end_lsn);
-extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferAbortOld(ReorderBuffer *, TransactionId xid);
-extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferInvalidate(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
extern void ReorderBufferSetBaseSnapshot(ReorderBuffer *, TransactionId, XLogRecPtr lsn, struct SnapshotData *snap);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..5be8f5755e 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -17,8 +17,11 @@
#include "access/xlogdefs.h"
#include "catalog/pg_subscription.h"
#include "datatype/timestamp.h"
+#include "replication/logicalrelation.h"
#include "storage/fileset.h"
#include "storage/lock.h"
+#include "storage/shm_mq.h"
+#include "storage/shm_toc.h"
#include "storage/spin.h"
@@ -60,6 +63,9 @@ typedef struct LogicalRepWorker
*/
FileSet *stream_fileset;
+ /* Indicates if this slot is used for an apply background worker. */
+ bool subworker;
+
/* Stats. */
XLogRecPtr last_lsn;
TimestampTz last_send_time;
@@ -68,8 +74,68 @@ typedef struct LogicalRepWorker
TimestampTz reply_time;
} LogicalRepWorker;
+/* Struct for saving and restoring apply errcontext information */
+typedef struct ApplyErrorCallbackArg
+{
+ LogicalRepMsgType command; /* 0 if invalid */
+ LogicalRepRelMapEntry *rel;
+
+ /* Remote node information */
+ int remote_attnum; /* -1 if invalid */
+ TransactionId remote_xid;
+ XLogRecPtr finish_lsn;
+ char *origin_name;
+} ApplyErrorCallbackArg;
+
+/*
+ * Status of apply background worker.
+ */
+typedef enum ApplyBgworkerStatus
+{
+ APPLY_BGWORKER_BUSY = 0, /* assigned to a transaction */
+ APPLY_BGWORKER_FINISHED, /* transaction is completed */
+ APPLY_BGWORKER_EXIT /* exit */
+} ApplyBgworkerStatus;
+
+/*
+ * Struct for sharing information between apply main and apply background
+ * workers.
+ */
+typedef struct ApplyBgworkerShared
+{
+ slock_t mutex;
+
+ /* Status of apply background worker. */
+ ApplyBgworkerStatus status;
+
+ /* server version of publisher. */
+ int server_version;
+
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
+
+/*
+ * Struct for maintaining an apply background worker.
+ */
+typedef struct ApplyBgworkerState
+{
+ shm_mq_handle *mq_handle;
+ dsm_segment *dsm_seg;
+ ApplyBgworkerShared volatile *pstate;
+} ApplyBgworkerState;
+
/* Main memory context for apply worker. Permanent during worker lifetime. */
extern PGDLLIMPORT MemoryContext ApplyContext;
+extern PGDLLIMPORT MemoryContext ApplyMessageContext;
+
+extern PGDLLIMPORT ApplyErrorCallbackArg apply_error_callback_arg;
+
+extern PGDLLIMPORT bool MySubscriptionValid;
+
+extern PGDLLIMPORT volatile ApplyBgworkerShared *MyParallelState;
+
+extern PGDLLIMPORT List *subxactlist;
/* libpqreceiver connection */
extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn;
@@ -79,18 +145,22 @@ extern PGDLLIMPORT Subscription *MySubscription;
extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker;
extern PGDLLIMPORT bool in_remote_transaction;
+extern PGDLLIMPORT bool in_streamed_transaction;
+extern PGDLLIMPORT TransactionId stream_xid;
extern void logicalrep_worker_attach(int slot);
extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
bool only_running);
extern List *logicalrep_workers_find(Oid subid, bool only_running);
-extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
- Oid userid, Oid relid);
+extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
+ Oid userid, Oid relid,
+ dsm_handle subworker_dsm);
extern void logicalrep_worker_stop(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
extern int logicalrep_sync_worker_count(Oid subid);
+extern int logicalrep_apply_bgworker_count(Oid subid);
extern void ReplicationOriginNameForTablesync(Oid suboid, Oid relid,
char *originname, int szorgname);
@@ -103,10 +173,38 @@ extern void process_syncing_tables(XLogRecPtr current_lsn);
extern void invalidate_syncing_table_states(Datum arg, int cacheid,
uint32 hashvalue);
+extern void UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time,
+ bool reply);
+
+extern void apply_dispatch(StringInfo s);
+
+/* Function for apply error callback */
+extern void apply_error_callback(void *arg);
+
+extern void subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue);
+
+/* Apply background worker setup and interactions */
+extern ApplyBgworkerState *apply_bgworker_start(TransactionId xid);
+extern ApplyBgworkerState *apply_bgworker_find(TransactionId xid);
+extern void apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status);
+extern void apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes,
+ const void *data);
+extern void apply_bgworker_free(ApplyBgworkerState *wstate);
+extern void apply_bgworker_check_status(void);
+extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
+extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+
static inline bool
am_tablesync_worker(void)
{
return OidIsValid(MyLogicalRepWorker->relid);
}
+static inline bool
+am_apply_bgworker(void)
+{
+ return MyLogicalRepWorker->subworker;
+}
+
#endif /* WORKER_INTERNAL_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b578e2ec75..c2d2a114d7 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -105,6 +105,7 @@ typedef enum
WAIT_EVENT_HASH_GROW_BUCKETS_ALLOCATE,
WAIT_EVENT_HASH_GROW_BUCKETS_ELECT,
WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE,
WAIT_EVENT_LOGICAL_SYNC_DATA,
WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE,
WAIT_EVENT_MQ_INTERNAL,
diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out
index 5db7146e06..919266ae06 100644
--- a/src/test/regress/expected/subscription.out
+++ b/src/test/regress/expected/subscription.out
@@ -197,7 +197,7 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
DROP SUBSCRIPTION regress_testsub;
-- fail - streaming must be boolean
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
-ERROR: streaming requires a Boolean value
+ERROR: streaming requires a Boolean value or "parallel"
-- now it works
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 34a76ceb60..4137dc77b4 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -116,6 +116,10 @@ Append
AppendPath
AppendRelInfo
AppendState
+ApplyBgworkerEntry
+ApplyBgworkerShared
+ApplyBgworkerState
+ApplyBgworkerStatus
ApplyErrorCallbackArg
ApplyExecutionData
ApplySubXactData
@@ -1485,6 +1489,7 @@ LogicalRepRelId
LogicalRepRelMapEntry
LogicalRepRelation
LogicalRepRollbackPreparedTxnData
+LogicalRepStreamAbortData
LogicalRepTupleData
LogicalRepTyp
LogicalRepWorker
--
2.23.0.windows.1
[application/octet-stream] v16-0002-Test-streaming-parallel-option-in-tap-test.patch (69.1K, ../../OS3PR01MB627594D75E870BBB45E2E80A9E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-v16-0002-Test-streaming-parallel-option-in-tap-test.patch)
download | inline diff:
From 882c87682d3a149ed27f4141c283f4262bc790f3 Mon Sep 17 00:00:00 2001
From: "shiy.fnst" <[email protected]>
Date: Fri, 13 May 2022 14:50:30 +0800
Subject: [PATCH v16 2/4] Test streaming parallel option in tap test
Change all TAP tests using the SUBSCRIPTION "streaming" parameter, so they
now test both 'on' and 'parallel' values.
---
src/test/subscription/t/015_stream.pl | 199 ++++---
src/test/subscription/t/016_stream_subxact.pl | 119 +++--
src/test/subscription/t/017_stream_ddl.pl | 188 ++++---
.../t/018_stream_subxact_abort.pl | 195 ++++---
.../t/019_stream_subxact_ddl_abort.pl | 110 +++-
.../subscription/t/022_twophase_cascade.pl | 363 +++++++------
.../subscription/t/023_twophase_stream.pl | 498 ++++++++++--------
7 files changed, 1035 insertions(+), 637 deletions(-)
diff --git a/src/test/subscription/t/015_stream.pl b/src/test/subscription/t/015_stream.pl
index 6561b189de..0bdd234935 100644
--- a/src/test/subscription/t/015_stream.pl
+++ b/src/test/subscription/t/015_stream.pl
@@ -8,6 +8,116 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Interleave a pair of transactions, each exceeding the 64kB limit.
+ my $in = '';
+ my $out = '';
+
+ my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
+
+ my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
+ on_error_stop => 0);
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ $in .= q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ };
+ $h->pump_nb;
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
+ DELETE FROM test_tab WHERE a > 5000;
+ COMMIT;
+ });
+
+ $in .= q{
+ COMMIT;
+ \q
+ };
+ $h->finish; # errors make the next test fail, so ignore them here
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'check extra columns contain local defaults');
+
+ # Test the streaming in binary mode
+ $node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain local defaults');
+
+ # Change the local values of the extra columns on the subscriber,
+ # update publisher, and check that subscriber retains the expected
+ # values. This is to ensure that non-streaming transactions behave
+ # properly after a streaming transaction.
+ $node_subscriber->safe_psql('postgres',
+ "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
+ );
+ $node_publisher->safe_psql('postgres',
+ "UPDATE test_tab SET b = md5(a::text)");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
+ );
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain locally changed data');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +147,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,82 +168,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Interleave a pair of transactions, each exceeding the 64kB limit.
-my $in = '';
-my $out = '';
-
-my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
-
-my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
- on_error_stop => 0);
-
-$in .= q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-};
-$h->pump_nb;
-
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
-DELETE FROM test_tab WHERE a > 5000;
-COMMIT;
-});
-
-$in .= q{
-COMMIT;
-\q
-};
-$h->finish; # errors make the next test fail, so ignore them here
-
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334), 'check extra columns contain local defaults');
-
-# Test the streaming in binary mode
-$node_subscriber->safe_psql('postgres',
- "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(6667|6667|6667), 'check extra columns contain local defaults');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+);
-# Change the local values of the extra columns on the subscriber,
-# update publisher, and check that subscriber retains the expected
-# values. This is to ensure that non-streaming transactions behave
-# properly after a streaming transaction.
$node_subscriber->safe_psql('postgres',
- "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
-);
-$node_publisher->safe_psql('postgres',
- "UPDATE test_tab SET b = md5(a::text)");
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel, binary = off)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
-);
-is($result, qq(6667|6667|6667),
- 'check extra columns contain locally changed data');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/016_stream_subxact.pl b/src/test/subscription/t/016_stream_subxact.pl
index f27f1694f2..45429dddba 100644
--- a/src/test/subscription/t/016_stream_subxact.pl
+++ b/src/test/subscription/t/016_stream_subxact.pl
@@ -8,6 +8,72 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(1667|1667|1667),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +103,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,41 +124,26 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Insert, update and delete enough rows to exceed 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(1667|1667|1667),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/017_stream_ddl.pl b/src/test/subscription/t/017_stream_ddl.pl
index 0bce63b716..52dfef4780 100644
--- a/src/test/subscription/t/017_stream_ddl.pl
+++ b/src/test/subscription/t/017_stream_ddl.pl
@@ -8,6 +8,111 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (3, md5(3::text));
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (4, md5(4::text), -4);
+ COMMIT;
+ });
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
+ is($result, qq(2002|1999|1002|1),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # A large (streamed) transaction with DDL and DML. One of the DDL is performed
+ # after DML to ensure that we invalidate the schema sent for test_tab so that
+ # the next transaction has to send the schema again.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
+ ALTER TABLE test_tab ADD COLUMN f INT;
+ COMMIT;
+ });
+
+ # A small transaction that won't get streamed. This is just to ensure that we
+ # send the schema again to reflect the last column added in the previous test.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab"
+ );
+ is($result, qq(5005|5002|4005|3004|5),
+ 'check data was copied to subscriber for both streaming and non-streaming transactions'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c, DROP COLUMN d, DROP COLUMN e, DROP COLUMN f;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +142,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,76 +163,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|0|0), 'check initial data was copied to subscriber');
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (3, md5(3::text));
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (4, md5(4::text), -4);
-COMMIT;
-});
-
-# large (streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
-COMMIT;
-});
-
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
-is($result, qq(2002|1999|1002|1),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# A large (streamed) transaction with DDL and DML. One of the DDL is performed
-# after DML to ensure that we invalidate the schema sent for test_tab so that
-# the next transaction has to send the schema again.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
-ALTER TABLE test_tab ADD COLUMN f INT;
-COMMIT;
-});
-
-# A small transaction that won't get streamed. This is just to ensure that we
-# send the schema again to reflect the last column added in the previous test.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
-COMMIT;
-});
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab");
-is($result, qq(5005|5002|4005|3004|5),
- 'check data was copied to subscriber for both streaming and non-streaming transactions'
-);
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/018_stream_subxact_abort.pl b/src/test/subscription/t/018_stream_subxact_abort.pl
index 7155442e76..68f0e4b0d1 100644
--- a/src/test/subscription/t/018_stream_subxact_abort.pl
+++ b/src/test/subscription/t/018_stream_subxact_abort.pl
@@ -8,6 +8,113 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
+ ROLLBACK TO s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
+ SAVEPOINT s5;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2000|0),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # large (streamed) transaction with subscriber receiving out of order
+ # subtransaction ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
+ RELEASE s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
+ ROLLBACK TO s1;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0),
+ 'check rollback to savepoint was reflected on subscriber');
+
+ # large (streamed) transaction with subscriber receiving rollback
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
+ ROLLBACK;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -36,6 +143,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -53,81 +164,25 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
-ROLLBACK TO s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
-SAVEPOINT s5;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2000|0),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# large (streamed) transaction with subscriber receiving out of order
-# subtransaction ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
-RELEASE s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
-ROLLBACK TO s1;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0),
- 'check rollback to savepoint was reflected on subscriber');
-
-# large (streamed) transaction with subscriber receiving rollback
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
-ROLLBACK;
-});
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
index dbd0fca4d1..b276063721 100644
--- a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
+++ b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
@@ -9,6 +9,69 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
+ ALTER TABLE test_tab DROP COLUMN c;
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(1000|500),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +100,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,35 +121,26 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
-ALTER TABLE test_tab DROP COLUMN c;
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(1000|500),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 7a797f37ba..0a4152d3be 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -11,6 +11,208 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming" parameter
+# so the same code can be run both for the streaming=on and streaming=parallel
+# cases.
+sub test_streaming
+{
+ my ($node_A, $node_B, $node_C, $appname_B, $appname_C, $streaming_mode) =
+ @_;
+
+ my $oldpid_B = $node_A->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';");
+ my $oldpid_C = $node_B->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';");
+
+ # Setup logical replication streaming mode
+
+ $node_B->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_B
+ SET (streaming = $streaming_mode);");
+ $node_C->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_C
+ SET (streaming = $streaming_mode)");
+
+ # Wait for subscribers to finish initialization
+
+ $node_A->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_B FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+ $node_B->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_C FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber(s) after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" optparameterion is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_B->reload;
+
+ $node_C->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_C->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ # Then 2PC PREPARE
+ $node_A->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_B->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_B->reload;
+
+ $node_C->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_C->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_C->reload;
+ }
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is prepared on subscriber(s)
+ my $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check that transaction was committed on subscriber(s)
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
+ );
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
+ );
+
+ # check the transaction state is ended on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber C');
+
+ ###############################
+ # Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
+ # 0. Cleanup from previous test leaving only 2 rows.
+ # 1. Insert one more row.
+ # 2. Record a SAVEPOINT.
+ # 3. Data is streamed using 2PC.
+ # 4. Do rollback to SAVEPOINT prior to the streamed inserts.
+ # 5. Then COMMIT PREPARED.
+ #
+ # Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (delete will be replicated)
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+
+ # 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
+ $node_A->safe_psql(
+ 'postgres', "
+ BEGIN;
+ INSERT INTO test_tab VALUES (9999, 'foobar');
+ SAVEPOINT sp_inner;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ ROLLBACK TO SAVEPOINT sp_inner;
+ PREPARE TRANSACTION 'outer';
+ ");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state prepared on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is ended on subscriber
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber C');
+
+ # check inserts are visible at subscriber(s).
+ # All the streamed data (prior to the SAVEPOINT) should be rolled back.
+ # (9999, 'foobar') should be committed.
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber B');
+ $result =
+ $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber B');
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber C');
+ $result =
+ $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber C');
+
+ # Cleanup the test data
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+}
+
###############################
# Setup a cascade of pub/sub nodes.
# node_A -> node_B -> node_C
@@ -260,160 +462,15 @@ is($result, qq(21), 'Rows committed are present on subscriber C');
# 2PC + STREAMING TESTS
# ---------------------
-my $oldpid_B = $node_A->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';");
-my $oldpid_C = $node_B->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';");
-
-# Setup logical replication (streaming = on)
-
-$node_B->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_B
- SET (streaming = on);");
-$node_C->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_C
- SET (streaming = on)");
-
-# Wait for subscribers to finish initialization
-
-$node_A->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_B FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-$node_B->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_C FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber(s) after the commit.
-###############################
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-# Then 2PC PREPARE
-$node_A->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
+################################
+# Test using streaming mode 'on'
+################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'on');
-# check the transaction state is prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check that transaction was committed on subscriber(s)
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
-);
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
-);
-
-# check the transaction state is ended on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber C');
-
-###############################
-# Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
-# 0. Cleanup from previous test leaving only 2 rows.
-# 1. Insert one more row.
-# 2. Record a SAVEPOINT.
-# 3. Data is streamed using 2PC.
-# 4. Do rollback to SAVEPOINT prior to the streamed inserts.
-# 5. Then COMMIT PREPARED.
-#
-# Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
-###############################
-
-# First, delete the data except for 2 rows (delete will be replicated)
-$node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
-$node_A->safe_psql(
- 'postgres', "
- BEGIN;
- INSERT INTO test_tab VALUES (9999, 'foobar');
- SAVEPOINT sp_inner;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- ROLLBACK TO SAVEPOINT sp_inner;
- PREPARE TRANSACTION 'outer';
- ");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state is ended on subscriber
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber C');
-
-# check inserts are visible at subscriber(s).
-# All the streamed data (prior to the SAVEPOINT) should be rolled back.
-# (9999, 'foobar') should be committed.
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber B');
-$result = $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber B');
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber C');
-$result = $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber C');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'parallel');
###############################
# check all the cleanup
diff --git a/src/test/subscription/t/023_twophase_stream.pl b/src/test/subscription/t/023_twophase_stream.pl
index d8475d25a4..b89414ab74 100644
--- a/src/test/subscription/t/023_twophase_stream.pl
+++ b/src/test/subscription/t/023_twophase_stream.pl
@@ -8,6 +8,266 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # check that 2PC gets replicated to subscriber
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ my $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ ###############################
+ # Test 2PC PREPARE / ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. Do rollback prepared.
+ #
+ # Expect data rolls back leaving only the original 2 rows.
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(2|2|2),
+ 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
+ # 1. insert, update and delete enough rows to exceed the 64kB limit.
+ # 2. Then server crashes before the 2PC transaction is committed.
+ # 3. After servers are restarted the pending transaction is committed.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ # Note: both publisher and subscriber do crash/restart.
+ ###############################
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_subscriber->stop('immediate');
+ $node_publisher->stop('immediate');
+
+ $node_publisher->start;
+ $node_subscriber->start;
+
+ # commit post the restart
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+ $node_publisher->wait_for_catchup($appname);
+
+ # check inserts are visible
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ ###############################
+ # Do INSERT after the PREPARE but before ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a ROLLBACK PREPARED.
+ #
+ # Expect the 2PC data rolls back leaving only 3 rows on the subscriber
+ # (the original 2 + inserted 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber,
+ # but the extra INSERT outside of the 2PC still was replicated
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3|3|3),
+ 'check the outside insert was copied to subscriber');
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Do INSERT after the PREPARE but before COMMIT PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a COMMIT PREPARED.
+ #
+ # Expect 2PC data + the extra row are on the subscriber
+ # (the 3334 + inserted 1 = 3335).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3335|3335|3335),
+ 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+ $node_publisher->wait_for_catchup($appname);
+}
+
###############################
# Setup
###############################
@@ -48,6 +308,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql(
'postgres', "
CREATE SUBSCRIPTION tap_sub
@@ -70,236 +334,30 @@ my $twophase_query =
$node_subscriber->poll_query_until('postgres', $twophase_query)
or die "Timed out while waiting for subscriber to enable twophase";
-###############################
# Check initial data was copied to subscriber
-###############################
my $result = $node_subscriber->safe_psql('postgres',
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber side after the commit.
-###############################
-
-# check that 2PC gets replicated to subscriber
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
-);
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
-
-###############################
-# Test 2PC PREPARE / ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. Do rollback prepared.
-#
-# Expect data rolls back leaving only the original 2 rows.
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(2|2|2),
- 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
-# 1. insert, update and delete enough rows to exceed the 64kB limit.
-# 2. Then server crashes before the 2PC transaction is committed.
-# 3. After servers are restarted the pending transaction is committed.
-#
-# Expect all data is replicated on subscriber side after the commit.
-# Note: both publisher and subscriber do crash/restart.
-###############################
-
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_subscriber->stop('immediate');
-$node_publisher->stop('immediate');
-
-$node_publisher->start;
-$node_subscriber->start;
-
-# commit post the restart
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-# check inserts are visible
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-###############################
-# Do INSERT after the PREPARE but before ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a ROLLBACK PREPARED.
-#
-# Expect the 2PC data rolls back leaving only 3 rows on the subscriber
-# (the original 2 + inserted 1).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber,
-# but the extra INSERT outside of the 2PC still was replicated
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3|3|3), 'check the outside insert was copied to subscriber');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Do INSERT after the PREPARE but before COMMIT PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a COMMIT PREPARED.
-#
-# Expect 2PC data + the extra row are on the subscriber
-# (the 3334 + inserted 1 = 3335).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3335|3335|3335),
- 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
-);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
###############################
# check all the cleanup
--
2.23.0.windows.1
[application/octet-stream] v16-0003-Add-some-checks-before-using-apply-background-wo.patch (36.6K, ../../OS3PR01MB627594D75E870BBB45E2E80A9E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-v16-0003-Add-some-checks-before-using-apply-background-wo.patch)
download | inline diff:
From fae06ce145c7f239eb27293999afcc44b04eebfb Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 11:23:52 +0800
Subject: [PATCH v16 3/4] Add some checks before using apply background worker
to apply changes.
streaming=parallel mode has two requirements:
1) The unique column in the relation on the subscriber-side should also be the
unique column on the publisher-side;
2) There cannot be any non-immutable functions in the subscriber-side
replicated table. Look for functions in the following places:
* a. Trigger functions
* b. Column default value expressions and domain constraints
* c. Constraint expressions
* d. Foreign keys
---
doc/src/sgml/ref/create_subscription.sgml | 4 +
.../replication/logical/applybgworker.c | 42 ++
src/backend/replication/logical/proto.c | 66 ++-
src/backend/replication/logical/relation.c | 199 +++++++++
src/backend/replication/logical/tablesync.c | 1 +
src/backend/replication/logical/worker.c | 23 +-
src/backend/utils/cache/typcache.c | 17 +
src/include/replication/logicalproto.h | 1 +
src/include/replication/logicalrelation.h | 15 +
src/include/replication/worker_internal.h | 1 +
src/include/utils/typcache.h | 2 +
.../subscription/t/022_twophase_cascade.pl | 6 +
.../subscription/t/032_streaming_apply.pl | 391 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
14 files changed, 759 insertions(+), 10 deletions(-)
create mode 100644 src/test/subscription/t/032_streaming_apply.pl
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 71dd4aca81..270e3d382e 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -240,6 +240,10 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
transaction is committed. Note that if an error happens when
applying changes in a background worker, the finish LSN of the
remote transaction might not be reported in the server log.
+ Parallel mode has two requirements: 1) the unique column in the
+ relation on the subscriber-side should also be the unique column on
+ the publisher-side; 2) there cannot be any non-immutable functions
+ in the subscriber-side replicated table.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index bb2b180e29..3be238223b 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -784,3 +784,45 @@ apply_bgworker_subxact_info_add(TransactionId current_xid)
MemoryContextSwitchTo(oldctx);
}
}
+
+/*
+ * Check if changes on this relation can be applied by an apply background
+ * worker.
+ *
+ * Although the commit order is maintained only allowing one process to commit
+ * at a time, the access order to the relation has changed. This could cause
+ * unexpected problems if the unique column on the replicated table is
+ * inconsistent with the publisher-side or contains non-immutable functions
+ * when applying transactions in the apply background worker.
+ */
+void
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
+{
+ /* Skip check if not an apply background worker. */
+ if (!am_apply_bgworker())
+ return;
+
+ /*
+ * Partition table checks are done later in function
+ * apply_handle_tuple_routing.
+ */
+ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ return;
+
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel == PARALLEL_APPLY_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot replicate target relation \"%s.%s\" in parallel "
+ "mode", rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change the streaming option to 'on' instead of 'parallel'.")));
+}
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index affd08cfa4..12a9799f5c 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -23,7 +23,8 @@
/*
* Protocol message flags.
*/
-#define LOGICALREP_IS_REPLICA_IDENTITY 1
+#define ATTR_IS_REPLICA_IDENTITY (1 << 0)
+#define ATTR_IS_UNIQUE (1 << 1)
#define MESSAGE_TRANSACTIONAL (1<<0)
#define TRUNCATE_CASCADE (1<<0)
@@ -933,11 +934,55 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
TupleDesc desc;
int i;
uint16 nliveatts = 0;
- Bitmapset *idattrs = NULL;
+ Bitmapset *idattrs = NULL,
+ *attunique = NULL;
bool replidentfull;
desc = RelationGetDescr(rel);
+ if (rel->rd_rel->relhasindex)
+ {
+ List *indexoidlist = RelationGetIndexList(rel);
+ ListCell *indexoidscan;
+
+ foreach(indexoidscan, indexoidlist)
+ {
+ Oid indexoid = lfirst_oid(indexoidscan);
+ Relation indexRel;
+
+ /* Look up the description for index */
+ indexRel = RelationIdGetRelation(indexoid);
+
+ if (!RelationIsValid(indexRel))
+ elog(ERROR, "could not open relation with OID %u", indexoid);
+
+ if (indexRel->rd_index->indisunique)
+ {
+ int i;
+
+ /* Add referenced attributes to idindexattrs */
+ for (i = 0; i < indexRel->rd_index->indnatts; i++)
+ {
+ int attrnum = indexRel->rd_index->indkey.values[i];
+
+ /*
+ * We don't include non-key columns into idindexattrs
+ * bitmaps. See RelationGetIndexAttrBitmap.
+ */
+ if (attrnum != 0)
+ {
+ if (i < indexRel->rd_index->indnkeyatts &&
+ !bms_is_member(attrnum - FirstLowInvalidHeapAttributeNumber, attunique))
+ attunique = bms_add_member(attunique,
+ attrnum - FirstLowInvalidHeapAttributeNumber);
+ }
+ }
+ }
+ RelationClose(indexRel);
+ }
+ list_free(indexoidlist);
+ }
+
/* send number of live attributes */
for (i = 0; i < desc->natts; i++)
{
@@ -974,7 +1019,11 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (replidentfull ||
bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
idattrs))
- flags |= LOGICALREP_IS_REPLICA_IDENTITY;
+ flags |= ATTR_IS_REPLICA_IDENTITY;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ attunique))
+ flags |= ATTR_IS_UNIQUE;
pq_sendbyte(out, flags);
@@ -1001,7 +1050,8 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
int natts;
char **attnames;
Oid *atttyps;
- Bitmapset *attkeys = NULL;
+ Bitmapset *attkeys = NULL,
+ *attunique = NULL;
natts = pq_getmsgint(in, 2);
attnames = palloc(natts * sizeof(char *));
@@ -1012,11 +1062,14 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
{
uint8 flags;
- /* Check for replica identity column */
+ /* Check for replica identity and unique column */
flags = pq_getmsgbyte(in);
- if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
+ if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
+ if (flags & ATTR_IS_UNIQUE)
+ attunique = bms_add_member(attunique, i);
+
/* attribute name */
attnames[i] = pstrdup(pq_getmsgstring(in));
@@ -1030,6 +1083,7 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
rel->attnames = attnames;
rel->atttyps = atttyps;
rel->attkeys = attkeys;
+ rel->attunique = attunique;
rel->natts = natts;
}
diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c
index e989047681..f4f5531e4d 100644
--- a/src/backend/replication/logical/relation.c
+++ b/src/backend/replication/logical/relation.c
@@ -19,12 +19,19 @@
#include "access/table.h"
#include "catalog/namespace.h"
+#include "catalog/pg_proc.h"
#include "catalog/pg_subscription_rel.h"
+#include "commands/trigger.h"
#include "executor/executor.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "replication/logicalrelation.h"
#include "replication/worker_internal.h"
+#include "rewrite/rewriteHandler.h"
#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
static MemoryContext LogicalRepRelMapContext = NULL;
@@ -91,6 +98,26 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
}
}
+/*
+ * Relcache invalidation callback to reset parallel flag.
+ */
+static void
+logicalrep_relmap_reset_parallel_cb(Datum arg, int cacheid, uint32 hashvalue)
+{
+ HASH_SEQ_STATUS hash_seq;
+ LogicalRepRelMapEntry *entry;
+
+ if (LogicalRepRelMap == NULL)
+ return;
+
+ hash_seq_init(&hash_seq, LogicalRepRelMap);
+ while ((entry = hash_seq_search(&hash_seq)) != NULL)
+ {
+ entry->parallel = PARALLEL_APPLY_UNKNOWN;
+ entry->localrelvalid = false;
+ }
+}
+
/*
* Initialize the relation map cache.
*/
@@ -116,6 +143,9 @@ logicalrep_relmap_init(void)
/* Watch for invalidation events. */
CacheRegisterRelcacheCallback(logicalrep_relmap_invalidate_cb,
(Datum) 0);
+ CacheRegisterSyscacheCallback(PROCOID,
+ logicalrep_relmap_reset_parallel_cb,
+ (Datum) 0);
}
/*
@@ -142,6 +172,7 @@ logicalrep_relmap_free_entry(LogicalRepRelMapEntry *entry)
pfree(remoterel->atttyps);
}
bms_free(remoterel->attkeys);
+ bms_free(remoterel->attunique);
if (entry->attrmap)
free_attrmap(entry->attrmap);
@@ -190,6 +221,7 @@ logicalrep_relmap_update(LogicalRepRelation *remoterel)
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
MemoryContextSwitchTo(oldctx);
}
@@ -310,6 +342,166 @@ logicalrep_rel_mark_updatable(LogicalRepRelMapEntry *entry)
}
}
+/*
+ * Check if changes on one relation can be applied by an apply background
+ * worker and assign the 'parallel' flag.
+ *
+ * There are two requirements for applying changes in an apply background
+ * worker: 1) The unique column in the relation on the subscriber-side should
+ * also be the unique column on the publisher-side; 2) There cannot be any
+ * non-immutable functions in the subscriber-side.
+ *
+ * We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
+ * on one relation can not be applied by an apply background worker and leave
+ * it to apply_bgworker_relation_check() to throw the actual error if needed.
+ */
+static void
+logicalrep_rel_mark_parallel(LogicalRepRelMapEntry *entry)
+{
+ Bitmapset *ukey;
+ int i;
+ TupleDesc tupdesc;
+ int attnum;
+ List *fkeys = NIL;
+
+ /* Fast path if we marked 'parallel' flag. */
+ if (entry->parallel != PARALLEL_APPLY_UNKNOWN)
+ return;
+
+ /* Initialize the flag. */
+ entry->parallel = PARALLEL_APPLY_SAFE;
+
+ /*
+ * First, we check if the unique column in the relation on the
+ * subscriber-side is also the unique column on the publisher-side.
+ */
+ ukey = RelationGetIndexAttrBitmap(entry->localrel,
+ INDEX_ATTR_BITMAP_KEY);
+
+ if (ukey)
+ {
+ i = -1;
+ while ((i = bms_next_member(ukey, i)) >= 0)
+ {
+ attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
+
+ if (entry->attrmap->attnums[attnum] < 0 ||
+ !bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
+ {
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /*
+ * Then, We check if there is any non-immutable function in the local
+ * table. Look for functions in the following places:
+ * a. trigger functions;
+ * b. Column default value expressions and domain constraints;
+ * c. Constraint expressions;
+ * d. Foreign keys.
+ */
+ /* Check the trigger functions. */
+ if (entry->localrel->trigdesc != NULL)
+ {
+ for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
+ {
+ Trigger *trig = entry->localrel->trigdesc->triggers + i;
+
+ if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
+ trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
+ continue;
+
+ if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
+ {
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the columns. */
+ tupdesc = RelationGetDescr(entry->localrel);
+ for (attnum = 0; attnum < tupdesc->natts; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
+
+ /* We don't need info for dropped or generated attributes */
+ if (att->attisdropped || att->attgenerated)
+ continue;
+
+ /*
+ * We don't need to check columns that only exist on the
+ * subscriber
+ */
+ if (entry->attrmap->attnums[attnum] < 0)
+ continue;
+
+ if (att->atthasdef)
+ {
+ Node *defaultexpr;
+
+ defaultexpr = build_column_default(entry->localrel, attnum + 1);
+ if (contain_mutable_functions(defaultexpr))
+ {
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+
+ /*
+ * If the column is of a DOMAIN type, determine whether
+ * that domain has any CHECK expressions that are not
+ * immutable.
+ */
+ if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
+ {
+ List *domain_constraints;
+ ListCell *lc;
+
+ domain_constraints = GetDomainConstraints(att->atttypid);
+
+ foreach(lc, domain_constraints)
+ {
+ DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
+
+ if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
+ {
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+ }
+
+ /* Check the constraints. */
+ if (tupdesc->constr)
+ {
+ ConstrCheck *check = tupdesc->constr->check;
+
+ /*
+ * Determine if there are any CHECK constraints which
+ * contains non-immutable function.
+ */
+ for (i = 0; i < tupdesc->constr->num_check; i++)
+ {
+ Expr *check_expr = stringToNode(check[i].ccbin);
+
+ if (contain_mutable_functions((Node *) check_expr))
+ {
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the foreign keys. */
+ fkeys = RelationGetFKeyList(entry->localrel);
+ if (fkeys)
+ entry->parallel = PARALLEL_APPLY_UNSAFE;
+}
+
/*
* Open the local relation associated with the remote one.
*
@@ -438,6 +630,9 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
*/
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel(entry);
+
entry->localrelvalid = true;
}
@@ -653,6 +848,7 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
}
entry->localrel = partrel;
@@ -696,6 +892,9 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
/* Set if the table's replica identity is enough to apply update/delete. */
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel(entry);
+
entry->localrelvalid = true;
/* state and statelsn are left set to 0. */
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8ffba7e2e5..3cdbf8b457 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -884,6 +884,7 @@ fetch_remote_table_info(char *nspname, char *relname,
lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
lrel->attkeys = NULL;
+ lrel->attunique = NULL;
/*
* Store the columns as a list of names. Ignore those that are not
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index bbd0304a8d..a46eb7dfab 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -1388,6 +1388,14 @@ apply_handle_stream_stop(StringInfo s)
{
char action = LOGICAL_REP_MSG_STREAM_STOP;
+ /*
+ * Unlike stream_commit, we don't need to wait here for stream_stop to
+ * finish. Allowing the other transaction to be applied before
+ * stream_stop is finished can lead to failures if the unique
+ * index/constraint is different between publisher and subscriber. But
+ * for such cases, we don't allow streamed transactions to be applied
+ * in parallel. See apply_bgworker_relation_check.
+ */
apply_bgworker_send_data(stream_apply_worker, 1, &action);
elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
@@ -2040,6 +2048,8 @@ apply_handle_insert(StringInfo s)
/* Set relation for error callback */
apply_error_callback_arg.rel = rel;
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2183,6 +2193,8 @@ apply_handle_update(StringInfo s)
/* Check if we can do the update. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2351,6 +2363,8 @@ apply_handle_delete(StringInfo s)
/* Check if we can do the delete. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2536,13 +2550,14 @@ apply_handle_tuple_routing(ApplyExecutionData *edata,
}
MemoryContextSwitchTo(oldctx);
+ part_entry = logicalrep_partition_open(relmapentry, partrel,
+ attrmap);
+
/* Check if we can do the update or delete on the leaf partition. */
if (operation == CMD_UPDATE || operation == CMD_DELETE)
- {
- part_entry = logicalrep_partition_open(relmapentry, partrel,
- attrmap);
check_relation_updatable(part_entry);
- }
+
+ apply_bgworker_relation_check(part_entry);
switch (operation)
{
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 808f9ebd0d..b248899d82 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -2540,6 +2540,23 @@ compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2)
return 0;
}
+/*
+ * GetDomainConstraints --- get DomainConstraintState list of specified domain type
+ */
+List *
+GetDomainConstraints(Oid type_id)
+{
+ TypeCacheEntry *typentry;
+ List *constraints = NIL;
+
+ typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
+
+ if(typentry->domainData != NULL)
+ constraints = typentry->domainData->constraints;
+
+ return constraints;
+}
+
/*
* Load (or re-load) the enumData member of the typcache entry.
*/
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index 0f74a3392b..130c4eb82f 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -113,6 +113,7 @@ typedef struct LogicalRepRelation
char replident; /* replica identity */
char relkind; /* remote relation kind */
Bitmapset *attkeys; /* Bitmap of key columns */
+ Bitmapset *attunique; /* Bitmap of unique columns */
} LogicalRepRelation;
/* Type mapping info */
diff --git a/src/include/replication/logicalrelation.h b/src/include/replication/logicalrelation.h
index 78cd7e77f5..7a1e9f762a 100644
--- a/src/include/replication/logicalrelation.h
+++ b/src/include/replication/logicalrelation.h
@@ -15,6 +15,19 @@
#include "access/attmap.h"
#include "replication/logicalproto.h"
+/*
+ * States to determine if changes on one relation can be applied by an apply
+ * background worker.
+ */
+typedef enum RelParallel
+{
+ PARALLEL_APPLY_UNKNOWN = 0, /* unknown */
+ PARALLEL_APPLY_SAFE, /* Can apply changes in an apply background
+ worker */
+ PARALLEL_APPLY_UNSAFE /* Can not apply changes in an apply background
+ worker */
+} RelParallel;
+
typedef struct LogicalRepRelMapEntry
{
LogicalRepRelation remoterel; /* key is remoterel.remoteid */
@@ -31,6 +44,8 @@ typedef struct LogicalRepRelMapEntry
Relation localrel; /* relcache entry (NULL when closed) */
AttrMap *attrmap; /* map of local attributes to remote ones */
bool updatable; /* Can apply updates/deletes? */
+ RelParallel parallel; /* Can apply changes in an apply
+ background worker? */
/* Sync state. */
char state;
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 5be8f5755e..b28f4ab977 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -194,6 +194,7 @@ extern void apply_bgworker_free(ApplyBgworkerState *wstate);
extern void apply_bgworker_check_status(void);
extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+extern void apply_bgworker_relation_check(LogicalRepRelMapEntry *rel);
static inline bool
am_tablesync_worker(void)
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index 431ad7f1b3..ed7c2e7f48 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -199,6 +199,8 @@ extern uint64 assign_record_type_identifier(Oid type_id, int32 typmod);
extern int compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2);
+extern List *GetDomainConstraints(Oid type_id);
+
extern size_t SharedRecordTypmodRegistryEstimate(void);
extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 0a4152d3be..30a01f7305 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -39,6 +39,12 @@ sub test_streaming
ALTER SUBSCRIPTION tap_sub_C
SET (streaming = $streaming_mode)");
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_C->safe_psql(
+ 'postgres', "ALTER TABLE test_tab ALTER c DROP DEFAULT");
+ }
+
# Wait for subscribers to finish initialization
$node_A->poll_query_until(
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
new file mode 100644
index 0000000000..eca4328676
--- /dev/null
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -0,0 +1,391 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test the restrictions of streaming mode "parallel" in logical replication
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $offset = 0;
+
+# Create publisher node
+my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->append_conf('postgresql.conf',
+ 'logical_decoding_work_mem = 64kB');
+$node_publisher->start;
+
+# Create subscriber node
+my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber');
+$node_subscriber->init;
+$node_subscriber->start;
+
+# Setup structure on publisher
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar)");
+
+# Setup structure on subscriber
+# We need to test normal table and partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar) PARTITION BY RANGE(a)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partition (LIKE test_tab_partitioned)");
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partitioned ATTACH PARTITION test_tab_partition DEFAULT"
+);
+
+# Setup logical replication
+my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub FOR TABLE test_tab");
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub_partitioned FOR TABLE test_tab_partitioned");
+
+my $appname = 'tap_sub';
+$node_subscriber->safe_psql(
+ 'postgres', "
+ CREATE SUBSCRIPTION tap_sub
+ CONNECTION '$publisher_connstr application_name=$appname'
+ PUBLICATION tap_pub, tap_pub_partitioned
+ WITH (streaming = parallel, copy_data = false)");
+
+$node_publisher->wait_for_catchup($appname);
+
+# It is not allowed that the unique index on the publisher and the subscriber
+# is different. Check the error reported by background worker in this case.
+# First we check the unique index on normal table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
+
+# Check that a background worker starts if "streaming" option is specified as
+# "parallel". We have to look for the DEBUG1 log messages about that, so
+# temporarily bump up the log verbosity.
+$node_subscriber->append_conf('postgresql.conf', "log_min_messages = debug1");
+$node_subscriber->reload;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+$node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+$node_subscriber->reload;
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+my $result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Then we check the unique index on partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_partition_idx ON test_tab_partition (b)");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Triggers which execute non-immutable function are not allowed on the
+# subscriber side. Check the error reported by background worker in this case.
+# First we check the trigger function on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE FUNCTION trigger_func() RETURNS TRIGGER AS \$\$
+ BEGIN
+ RETURN NULL;
+ END
+\$\$ language plpgsql;
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# Then we check the trigger function on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab_partition
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab_partition ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# It is not allowed that column default value expression contains a
+# non-immutable function on the subscriber side. Check the error reported by
+# background worker in this case.
+# First we check the column default value expression on normal table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# Then we check the column default value expression on partition table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# It is not allowed that domain constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case.
+# Because the column type of the partition table must be the same as its parent
+# table, only test normal table here.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE DOMAIN test_domain AS int CHECK(VALUE > random());
+ALTER TABLE test_tab ALTER COLUMN a TYPE test_domain;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop domain constraint expression on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping domain constraint expression'
+);
+
+# It is not allowed that constraint expression contains a non-immutable function
+# on the subscriber side. Check the error reported by background worker in this
+# case.
+# First we check the constraint expression on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# Then we check the constraint expression on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# It is not allowed that foreign key on the subscriber side. Check the error
+# reported by background worker in this case.
+# First we check the foreign key on normal table.
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_f (a int primary key);
+ALTER TABLE test_tab ADD CONSTRAINT test_tabfk FOREIGN KEY(a) REFERENCES test_tab_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+# Then we check the foreign key on partition table.
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_partition_f (a int primary key);
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_patition_fk FOREIGN KEY(a) REFERENCES test_tab_partition_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+$node_subscriber->stop;
+$node_publisher->stop;
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 4137dc77b4..ae7cd99159 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2271,6 +2271,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelParallel
RelToCheck
RelToCluster
RelabelType
--
2.23.0.windows.1
[application/octet-stream] v16-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch (26.6K, ../../OS3PR01MB627594D75E870BBB45E2E80A9E839@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-v16-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch)
download | inline diff:
From abcc64adb1cf721c8828ad0722e7cc1e25e44963 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 15 Jun 2022 11:02:08 +0800
Subject: [PATCH v16 4/4] Retry to apply streaming xact only in apply worker
If the user sets the subscription_parameter "streaming" to "parallel", when
applying a streaming transaction, we will try to apply this transaction in
apply background worker. However, when the changes in this transaction cannot
be applied in apply background worker, the background worker will exit with an
error. In this case, we can retry applying this streaming transaction in "on"
mode. In this way, we may avoid blocking logical replication here.
So we introduce field "subretry" in catalog "pg_subscription". When the
subscriber exit with an error, we will try to set this flag to true, and when
the transaction is applied successfully, we will try to set this flag to false.
Then when we try to apply a streaming transaction in apply background worker,
we can see if this transaction has failed before based on the "subretry" field.
---
doc/src/sgml/catalogs.sgml | 9 +
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/catalog/pg_subscription.c | 1 +
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/subscriptioncmds.c | 1 +
.../replication/logical/applybgworker.c | 15 +-
src/backend/replication/logical/worker.c | 89 ++++++++++
src/bin/pg_dump/pg_dump.c | 5 +-
src/include/catalog/pg_subscription.h | 4 +
.../subscription/t/032_streaming_apply.pl | 154 +++++++++++-------
10 files changed, 221 insertions(+), 66 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 815cae6082..28f3d121d9 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7907,6 +7907,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>subretry</structfield> <type>bool</type>
+ </para>
+ <para>
+ True if the previous apply change failed and a retry was required.
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>subconninfo</structfield> <type>text</type>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 270e3d382e..bd5361991e 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -244,6 +244,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
relation on the subscriber-side should also be the unique column on
the publisher-side; 2) there cannot be any non-immutable functions
in the subscriber-side replicated table.
+ When applying a streaming transaction, if either requirement is not
+ met, the background worker will exit with an error.
+ <literal>parallel</literal> mode is disregarded when retrying;
+ instead the transaction will be applied using <literal>on</literal>
+ mode.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index 8856ce3b50..9b7f09653d 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok)
sub->stream = subform->substream;
sub->twophasestate = subform->subtwophasestate;
sub->disableonerr = subform->subdisableonerr;
+ sub->retry = subform->subretry;
/* Get conninfo */
datum = SysCacheGetAttr(SUBSCRIPTIONOID,
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fedaed533b..10f4dd6785 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1298,8 +1298,8 @@ REVOKE ALL ON pg_replication_origin_status FROM public;
-- All columns of pg_subscription except subconninfo are publicly readable.
REVOKE ALL ON pg_subscription FROM public;
GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled,
- subbinary, substream, subtwophasestate, subdisableonerr, subslotname,
- subsynccommit, subpublications)
+ subbinary, substream, subtwophasestate, subdisableonerr,
+ subretry, subslotname, subsynccommit, subpublications)
ON pg_subscription TO public;
CREATE VIEW pg_stat_subscription_stats AS
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 5f349067cc..33aef31b30 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -662,6 +662,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
LOGICALREP_TWOPHASE_STATE_PENDING :
LOGICALREP_TWOPHASE_STATE_DISABLED);
values[Anum_pg_subscription_subdisableonerr - 1] = BoolGetDatum(opts.disableonerr);
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(false);
values[Anum_pg_subscription_subconninfo - 1] =
CStringGetTextDatum(conninfo);
if (opts.slot_name)
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index 3be238223b..5cfa8d1dbc 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -106,6 +106,18 @@ apply_bgworker_can_start(TransactionId xid)
if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
return false;
+ /*
+ * Don't use apply background workers for retries, because it is possible
+ * that the last time we tried to apply a transaction using an apply
+ * background worker the checks failed (see function
+ * apply_bgworker_relation_check).
+ */
+ if (MySubscription->retry)
+ {
+ elog(DEBUG1, "apply background workers are not used for retries");
+ return false;
+ }
+
/*
* For streaming transactions that are being applied in apply background
* worker, we cannot decide whether to apply the change for a relation
@@ -823,6 +835,5 @@ apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
"mode", rel->remoterel.nspname, rel->remoterel.relname),
errdetail("The unique column on subscriber is not the unique "
"column on publisher or there is at least one "
- "non-immutable function."),
- errhint("Please change the streaming option to 'on' instead of 'parallel'.")));
+ "non-immutable function.")));
}
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index a46eb7dfab..0daeb4f9c5 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -378,6 +378,8 @@ static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
+static void set_subscription_retry(bool retry);
+
/*
* Should this worker apply changes for given relation.
*
@@ -904,6 +906,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1015,6 +1020,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1068,6 +1076,9 @@ apply_handle_commit_prepared(StringInfo s)
store_flush_position(prepare_data.end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1123,6 +1134,9 @@ apply_handle_rollback_prepared(StringInfo s)
store_flush_position(rollback_data.rollback_end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(rollback_data.rollback_end_lsn);
@@ -1215,6 +1229,9 @@ apply_handle_stream_prepare(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
in_remote_transaction = false;
@@ -1642,6 +1659,9 @@ apply_handle_stream_abort(StringInfo s)
*/
serialize_stream_abort(xid, subxid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
reset_apply_error_context_info();
@@ -1854,6 +1874,9 @@ apply_handle_stream_commit(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
/* Check the status of apply background worker if any. */
@@ -3897,6 +3920,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
}
PG_CATCH();
{
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
else
@@ -3935,6 +3961,9 @@ start_apply(XLogRecPtr origin_startpos)
}
PG_CATCH();
{
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
else
@@ -4461,3 +4490,63 @@ reset_apply_error_context_info(void)
apply_error_callback_arg.remote_attnum = -1;
set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
}
+
+/*
+ * Set subretry of pg_subscription catalog.
+ *
+ * If retry is true, subscriber is about to exit with an error. Otherwise, it
+ * means that the transaction was applied successfully.
+ */
+static void
+set_subscription_retry(bool retry)
+{
+ Relation rel;
+ HeapTuple tup;
+ bool started_tx = false;
+ bool nulls[Natts_pg_subscription];
+ bool replaces[Natts_pg_subscription];
+ Datum values[Natts_pg_subscription];
+
+ if (MySubscription->retry == retry ||
+ am_apply_bgworker())
+ return;
+
+ if (!IsTransactionState())
+ {
+ StartTransactionCommand();
+ started_tx = true;
+ }
+
+ /* Look up the subscription in the catalog */
+ rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+ tup = SearchSysCacheCopy1(SUBSCRIPTIONOID,
+ ObjectIdGetDatum(MySubscription->oid));
+
+ if (!HeapTupleIsValid(tup))
+ elog(ERROR, "subscription \"%s\" does not exist", MySubscription->name);
+
+ LockSharedObject(SubscriptionRelationId, MySubscription->oid, 0,
+ AccessShareLock);
+
+ /* Form a new tuple. */
+ memset(values, 0, sizeof(values));
+ memset(nulls, false, sizeof(nulls));
+ memset(replaces, false, sizeof(replaces));
+
+ /* reset subretry */
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(retry);
+ replaces[Anum_pg_subscription_subretry - 1] = true;
+
+ tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+ replaces);
+
+ /* Update the catalog. */
+ CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+ /* Cleanup. */
+ heap_freetuple(tup);
+ table_close(rel, NoLock);
+
+ if (started_tx)
+ CommitTransactionCommand();
+}
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 24927641b9..9ce774fc39 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4471,8 +4471,9 @@ getSubscriptions(Archive *fout)
ntups = PQntuples(res);
/*
- * Get subscription fields. We don't include subskiplsn in the dump as
- * after restoring the dump this value may no longer be relevant.
+ * Get subscription fields. We don't include subskiplsn and subretry in
+ * the dump as after restoring the dump this value may no longer be
+ * relevant.
*/
i_tableoid = PQfnumber(res, "tableoid");
i_oid = PQfnumber(res, "oid");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d54540f5f5..5f4e058ec1 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -76,6 +76,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subdisableonerr; /* True if a worker error should cause the
* subscription to be disabled */
+ bool subretry BKI_DEFAULT(f); /* True if the previous apply change failed. */
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* Connection string to the publisher */
text subconninfo BKI_FORCE_NOT_NULL;
@@ -116,6 +118,8 @@ typedef struct Subscription
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
* occurs */
+ bool retry; /* Indicates if the previous apply change
+ * failed. */
char *conninfo; /* Connection string to the publisher */
char *slotname; /* Name of the replication slot */
char *synccommit; /* Synchronous commit setting for worker */
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
index eca4328676..1bcb4c65b7 100644
--- a/src/test/subscription/t/032_streaming_apply.pl
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -57,8 +57,13 @@ $node_subscriber->safe_psql(
$node_publisher->wait_for_catchup($appname);
+# ============================================================================
# It is not allowed that the unique index on the publisher and the subscriber
-# is different. Check the error reported by background worker in this case.
+# is different. Check the error reported by background worker in this case. And
+# after retrying in apply worker, we check if the data is replicated
+# successfully.
+# ============================================================================
+
# First we check the unique index on normal table.
$node_subscriber->safe_psql('postgres',
"CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
@@ -82,14 +87,15 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
my $result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
# Then we check the unique index on partition table.
$node_subscriber->safe_psql('postgres',
@@ -106,17 +112,20 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
# Triggers which execute non-immutable function are not allowed on the
# subscriber side. Check the error reported by background worker in this case.
+# And after retrying in apply worker, we check if the data is replicated
+# successfully.
# First we check the trigger function on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -140,15 +149,16 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
# Then we check the trigger function on partition table.
$node_subscriber->safe_psql(
@@ -168,19 +178,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab_partition");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+# ============================================================================
# It is not allowed that column default value expression contains a
# non-immutable function on the subscriber side. Check the error reported by
-# background worker in this case.
+# background worker in this case. And after retrying in apply worker, we check
+# if the data is replicated successfully.
+# ============================================================================
+
# First we check the column default value expression on normal table.
$node_subscriber->safe_psql('postgres',
"ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
@@ -196,16 +211,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
# Then we check the column default value expression on partition table.
$node_subscriber->safe_psql('postgres',
@@ -222,20 +238,25 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+# ============================================================================
# It is not allowed that domain constraint expression contains a non-immutable
# function on the subscriber side. Check the error reported by background
-# worker in this case.
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# Because the column type of the partition table must be the same as its parent
# table, only test normal table here.
$node_subscriber->safe_psql(
@@ -253,21 +274,26 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop domain constraint expression on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(0),
- 'data replicated to subscriber after dropping domain constraint expression'
+ 'data replicated to subscribers after retrying because of domain'
);
-# It is not allowed that constraint expression contains a non-immutable function
-# on the subscriber side. Check the error reported by background worker in this
-# case.
+# Drop domain constraint expression on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+# ============================================================================
+# It is not allowed that constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# First we check the constraint expression on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -285,16 +311,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
# Then we check the constraint expression on partition table.
$node_subscriber->safe_psql(
@@ -311,19 +338,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(0),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+# ============================================================================
# It is not allowed that foreign key on the subscriber side. Check the error
-# reported by background worker in this case.
+# reported by background worker in this case. And after retrying in apply
+# worker, we check if the data is replicated successfully.
+# ============================================================================
+
# First we check the foreign key on normal table.
$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
$node_publisher->wait_for_catchup($appname);
@@ -344,16 +376,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" in parallel mode/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
# Then we check the foreign key on partition table.
$node_publisher->wait_for_catchup($appname);
@@ -374,16 +407,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" in parallel mode/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
$node_subscriber->stop;
$node_publisher->stop;
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-13 04:33 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 43+ messages in thread
From: Peter Smith @ 2022-07-13 04:33 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
Below are my review comments for the v16* patch set:
========
v16-0001
========
1.0 <general>
There are places (comments, docs, errmsgs, etc) in the patch referring
to "parallel mode". I think every one of those references should be
found and renamed to "parallel streaming mode" or "streaming=parallel"
or at the very least match sure that "streaming" is in the same
sentence. IMO it's too vague just saying "parallel" without also
saying the context is for the "streaming" parameter.
I have commented on some of those examples below, but please search
everything anyway (including the docs) to catch the ones I haven't
explicitly mentioned.
======
1.1 src/backend/commands/subscriptioncmds.c
+defGetStreamingMode(DefElem *def)
+{
+ /*
+ * If no value given, assume "true" is meant.
+ */
Please fix this comment to identical to this pushed patch [1]
======
1.2 .../replication/logical/applybgworker.c - apply_bgworker_start
+ if (list_length(ApplyWorkersFreeList) > 0)
+ {
+ wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
+ ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
+ Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
+ }
The Assert that the entries in the free-list are FINISHED seems like
unnecessary checking. IIUC, code is already doing the Assert that
entries are FINISHED before allowing them into the free-list in the
first place.
~~~
1.3 .../replication/logical/applybgworker.c - apply_bgworker_find
+ if (found)
+ {
+ char status = entry->wstate->pstate->status;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ entry->wstate->pstate->n,
+ entry->wstate->pstate->stream_xid)));
+
+ Assert(status == APPLY_BGWORKER_BUSY);
+
+ return entry->wstate;
+ }
Why not remove that Assert but change the condition to be:
if (status != APPLY_BGWORKER_BUSY)
ereport(...)
======
1.4 src/backend/replication/logical/proto.c - logicalrep_write_stream_abort
@@ -1163,31 +1163,56 @@ logicalrep_read_stream_commit(StringInfo in,
LogicalRepCommitData *commit_data)
/*
* Write STREAM ABORT to the output stream. Note that xid and subxid will be
* same for the top-level transaction abort.
+ *
+ * If write_abort_lsn is true, send the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
"Otherwise not." -> ", otherwise don't."
~~~
1.5 src/backend/replication/logical/proto.c - logicalrep_read_stream_abort
+ *
+ * If read_abort_lsn is true, try to read the abort_lsn and abort_time fields.
+ * Otherwise not.
*/
void
-logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid)
+logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn)
"Otherwise not." -> ", otherwise don't."
======
1.6 src/backend/replication/logical/worker.c - file comment
+ * If streaming = parallel, We assign a new apply background worker (if
+ * available) as soon as the xact's first stream is received. The main apply
"We" -> "we" ... or maybe better just remove it completely.
~~~
1.7 src/backend/replication/logical/worker.c - apply_handle_stream_prepare
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
The comment should be changed how you had suggested [2], so that it
will be formatted the same way as a couple of other similar comments.
~~~
1.8 src/backend/replication/logical/worker.c - apply_handle_stream_abort
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelState->server_version >= 160000;
This is handling decisions about read/write of the protocol bytes. I
think feel like it will be better to be checking the server *protocol*
version (not the server postgres version) to make this decision – e.g.
this code should be using the new macro you introduced so it will end
up looking much like how the pgoutput_stream_abort code is doing it.
~~~
1.9 src/backend/replication/logical/worker.c - store_flush_position
@@ -2636,6 +2999,10 @@ store_flush_position(XLogRecPtr remote_lsn)
{
FlushPosition *flushpos;
+ /* We only need to collect the LSN in main apply worker */
+ if (am_apply_bgworker())
+ return;
+
SUGGESTION
/* Skip if not the main apply worker */
======
1.10 src/backend/replication/pgoutput/pgoutput.c
@@ -1820,6 +1820,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
XLogRecPtr abort_lsn)
{
ReorderBufferTXN *toptxn;
+ bool write_abort_lsn = false;
+ PGOutputData *data = (PGOutputData *) ctx->output_plugin_private;
/*
* The abort should happen outside streaming block, even for streamed
@@ -1832,8 +1834,13 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
Assert(rbtxn_is_streamed(toptxn));
+ /* We only send abort_lsn and abort_time if the subscriber needs them. */
+ if (data->protocol_version >= LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM)
+ write_abort_lsn = true;
+
IMO it's simpler to remove the declaration default assignment, and
instead this code can be written as:
write_abort_lsn = data->protocol_version >=
LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM;
======
1.11 src/include/replication/logicalproto.h
+ *
+ * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version
+ * with support for streaming large transactions in apply background worker.
+ * Introduced in PG16.
"in apply background worker" -> "using apply background workers"
~~~
1.12
+extern void logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool include_abort_lsn);
I think the "include_abort_lsn" is now renamed to "include_abort_lsn".
========
v16-0002
========
No comments.
========
v16-0003
========
3.0 <general>
Same comment about "parallel mode" as in comment #1.0
======
3.1 doc/src/sgml/ref/create_subscription.sgml
+ the publisher-side; 2) there cannot be any non-immutable functions
+ in the subscriber-side replicated table.
The functions are not table data so maybe it's better to say
"functions in the ..." -> "functions used by the ...". If you change
this then there are equivalent comments and commit messages that
should change to match it.
======
3.2 .../replication/logical/applybgworker.c - apply_bgworker_relation_check
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot replicate target relation \"%s.%s\" in parallel "
+ "mode", rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change the streaming option to 'on' instead of
'parallel'.")));
3.2a
SUGGESTED errmsg
"cannot replicate target relation \"%s.%s\" using subscription
parameter streaming=parallel"
3.2b
SUGGESTED errhint
"Please change to use subscription parameter streaming=on"
3.3
The errcode seems the wrong one. Perhaps it should be
ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE.
======
3.4 src/backend/replication/logical/proto.c - logicalrep_write_attrs
In [3] you wrote:
I think the file relcache.c should contain cache-build operations, and the code
I added doesn't have this operation. So I didn't change.
But I only gave relcache.c as an example. It can also be a new static
function in this same file, but anyway I still think this big slab of
code might be better if not done inline in logicalrep_write_attrs.
~~~
3.5 src/backend/replication/logical/proto.c - logicalrep_read_attrs
@@ -1012,11 +1062,14 @@ logicalrep_read_attrs(StringInfo in,
LogicalRepRelation *rel)
{
uint8 flags;
- /* Check for replica identity column */
+ /* Check for replica identity and unique column */
flags = pq_getmsgbyte(in);
- if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
+ if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
+ if (flags & ATTR_IS_UNIQUE)
+ attunique = bms_add_member(attunique, i);
The code comment really applies to all 3 statements so maybe better
not to have the blank line here.
======
3.6 src/backend/replication/logical/relation.c - logicalrep_rel_mark_parallel
3.6.a
+ /* Fast path if we marked 'parallel' flag. */
+ if (entry->parallel != PARALLEL_APPLY_UNKNOWN)
+ return;
SUGGESTED
Fast path if 'parallel' flag is already known.
~
3.6.b
+ /* Initialize the flag. */
+ entry->parallel = PARALLEL_APPLY_SAFE;
I think it makes more sense if assigning SAFE is the very *last* thing
this function does – not the first thing.
~
3.6.c
+ /*
+ * First, we check if the unique column in the relation on the
+ * subscriber-side is also the unique column on the publisher-side.
+ */
"First, we check..." -> "First, check..."
~
3.6.d
+ /*
+ * Then, We check if there is any non-immutable function in the local
+ * table. Look for functions in the following places:
"Then, We check..." -> "Then, check"
~~~
3.7 src/backend/replication/logical/relation.c - logicalrep_rel_mark_parallel
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-19 02:28 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 43+ messages in thread
From: [email protected] @ 2022-07-19 02:28 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jul 13, 2022 at 13:49 PM Peter Smith <[email protected]> wrote:
> Below are my review comments for the v16* patch set:
Thanks for your comments.
> ========
> v16-0001
> ========
>
> 1.0 <general>
>
> There are places (comments, docs, errmsgs, etc) in the patch referring
> to "parallel mode". I think every one of those references should be
> found and renamed to "parallel streaming mode" or "streaming=parallel"
> or at the very least match sure that "streaming" is in the same
> sentence. IMO it's too vague just saying "parallel" without also
> saying the context is for the "streaming" parameter.
>
> I have commented on some of those examples below, but please search
> everything anyway (including the docs) to catch the ones I haven't
> explicitly mentioned.
I checked all places in the patch where the word "parallel" is used (case
insensitive), and I think it is clear that the description is related to stream
transactions. So I am not so sure. Could you please give me some examples? I
will improve them later.
> 1.2 .../replication/logical/applybgworker.c - apply_bgworker_start
>
> + if (list_length(ApplyWorkersFreeList) > 0)
> + {
> + wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
> + ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
> + Assert(wstate->pstate->status == APPLY_BGWORKER_FINISHED);
> + }
>
> The Assert that the entries in the free-list are FINISHED seems like
> unnecessary checking. IIUC, code is already doing the Assert that
> entries are FINISHED before allowing them into the free-list in the
> first place.
Just for robustness.
> 1.3 .../replication/logical/applybgworker.c - apply_bgworker_find
>
> + if (found)
> + {
> + char status = entry->wstate->pstate->status;
> +
> + /* If any workers (or the postmaster) have died, we have failed. */
> + if (status == APPLY_BGWORKER_EXIT)
> + ereport(ERROR,
> + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> + errmsg("background worker %u failed to apply transaction %u",
> + entry->wstate->pstate->n,
> + entry->wstate->pstate->stream_xid)));
> +
> + Assert(status == APPLY_BGWORKER_BUSY);
> +
> + return entry->wstate;
> + }
>
> Why not remove that Assert but change the condition to be:
>
> if (status != APPLY_BGWORKER_BUSY)
> ereport(...)
When I check "APPLY_BGWORKER_EXIT", I use the function "ereport" to report the
error, because "APPLY_BGWORKER_EXIT" is a possible use case.
But for "APPLY_BGWORKER_BUSY", this use case should not happen here. So I think
it's fine to only check this for developers when the compile option
"--enable-cassert" is specified.
> ========
> v16-0003
> ========
>
> 3.0 <general>
>
> Same comment about "parallel mode" as in comment #1.0
>
> ======
Please refer to the reply to #1.0.
> 3.5 src/backend/replication/logical/proto.c - logicalrep_read_attrs
>
> @@ -1012,11 +1062,14 @@ logicalrep_read_attrs(StringInfo in,
> LogicalRepRelation *rel)
> {
> uint8 flags;
>
> - /* Check for replica identity column */
> + /* Check for replica identity and unique column */
> flags = pq_getmsgbyte(in);
> - if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
> + if (flags & ATTR_IS_REPLICA_IDENTITY)
> attkeys = bms_add_member(attkeys, i);
>
> + if (flags & ATTR_IS_UNIQUE)
> + attunique = bms_add_member(attunique, i);
>
> The code comment really applies to all 3 statements so maybe better
> not to have the blank line here.
I think it looks a bit messy without the blank line.
So I tried to improve it to the following:
```
/* Check for replica identity column */
flags = pq_getmsgbyte(in);
if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
/* Check for unique column */
if (flags & ATTR_IS_UNIQUE)
attunique = bms_add_member(attunique, i);
```
> 3.6 src/backend/replication/logical/relation.c - logicalrep_rel_mark_parallel
>
> 3.6.a
> + /* Fast path if we marked 'parallel' flag. */
> + if (entry->parallel != PARALLEL_APPLY_UNKNOWN)
> + return;
>
> SUGGESTED
> Fast path if 'parallel' flag is already known.
>
> ~
>
> 3.6.b
> + /* Initialize the flag. */
> + entry->parallel = PARALLEL_APPLY_SAFE;
>
> I think it makes more sense if assigning SAFE is the very *last* thing
> this function does – not the first thing.
>
> ~
>
> 3.6.c
> + /*
> + * First, we check if the unique column in the relation on the
> + * subscriber-side is also the unique column on the publisher-side.
> + */
>
> "First, we check..." -> "First, check..."
>
> ~
>
> 3.6.d
> + /*
> + * Then, We check if there is any non-immutable function in the local
> + * table. Look for functions in the following places:
>
>
> "Then, We check..." -> "Then, check"
=>3.6.a
=>3.6.c
=>3.6.d
Improved as suggested.
=>3.6.b
Not sure about this.
> 3.7 src/backend/replication/logical/relation.c - logicalrep_rel_mark_parallel
>
> From [3] you wrote:
> Personally, I do not like to use the `goto` syntax if it is not necessary,
> because the `goto` syntax will forcibly change the flow of code execution.
>
> Yes, but OTOH readability is a major consideration too, and in this
> function by simply saying goto parallel_unsafe; you can have 3 returns
> instead of 7 returns, and it will take ~10 lines less code to do the
> same functionality.
I am still not sure about this, I think I will change this if some more people
think `goto` is better here.
> 4.3 doc/src/sgml/ref/create_subscription.sgml
>
> + <literal>parallel</literal> mode is disregarded when retrying;
> + instead the transaction will be applied using <literal>on</literal>
> + mode.
>
> "on mode" etc sounds strange.
>
> SUGGESTION
> During the retry the streaming=parallel mode is ignored. The retried
> transaction will be applied using streaming=on mode.
Since it's part of the streaming option document. I think it's fine to directly
say "<literal>parallel</literal> mode"
> 4.4 src/backend/replication/logical/worker.c - set_subscription_retry
>
> + if (MySubscription->retry == retry ||
> + am_apply_bgworker())
> + return;
> +
>
> Somehow I feel that this quick exit condition is not quite what it
> seems. IIUC the purpose of this is really to avoid doing the tuple
> updates if it is not necessary to do them. So if retry was already set
> true then there is no need to update tuple to true again. So if retry
> was already set false then there is no need to update the tuple to
> false. But I just don't see how the (hypothetical) code below can work
> as expected, because where is the code updating the value of
> MySubscription->retry ???
>
> set_subscription_retry(true);
> set_subscription_retry(true);
>
> I think at least there needs to be some detailed comments explaining
> what this quick exit is really doing because my guess is that
> currently it is not quite working as expected.
The subscription cache is be updated in maybe_reread_subscription() and is
invoked at every transaction. And we reset the retry flag at transaction end,
so it should be fine. And I think the quick exit check code is similar to
clear_subscription_skip_lsn.
Attach the news patches.
[1] - https://www.postgresql.org/message-id/CAHut%2BPv0yWynWTmp4o34s0d98xVubys9fy%3Dp0YXsZ5_sUcNnMw%40mail...
Regards,
Wang wei
Attachments:
[application/octet-stream] v18-0001-Perform-streaming-logical-transactions-by-backgr.patch (106.6K, ../../OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-v18-0001-Perform-streaming-logical-transactions-by-backgr.patch)
download | inline diff:
From b0530ad3660d5571b56478100c94b702307cd09e Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Wed, 20 Apr 2022 16:45:07 +0800
Subject: [PATCH v18 1/4] Perform streaming logical transactions by background
workers
Currently, for large transactions, the publisher sends the data in multiple
streams (changes divided into chunks depending upon logical_decoding_work_mem),
and then on the subscriber-side, the apply worker writes the changes into
temporary files and once it receives the commit, it reads from the file and
applies the entire transaction. To improve the performance of such
transactions, we can instead allow them to be applied via background workers.
In this approach, we assign a new apply background worker (if available) as
soon as the xact's first stream is received and the main apply worker will send
changes to this new worker via shared memory. The apply background worker will
directly apply the change instead of writing it to temporary files. We keep
this worker assigned till the transaction commit is received and also wait for
the worker to finish at commit. This preserves commit ordering and avoids
writing to and reading from file in most cases. We still need to spill if there
is no worker available.
This patch also extends the SUBSCRIPTION 'streaming' parameter so that the user
can control whether to apply the streaming transaction in an apply background
worker or spill the change to disk. The user can set the streaming parameter to
'on/off', 'parallel'. The parameter value 'parallel' means the streaming will
be applied via an apply background worker, if available. The parameter value
'on' means the streaming transaction will be spilled to disk. The default value
is 'off' (same as current behaviour).
---
doc/src/sgml/catalogs.sgml | 10 +-
doc/src/sgml/config.sgml | 25 +
doc/src/sgml/logical-replication.sgml | 10 +
doc/src/sgml/protocol.sgml | 19 +
doc/src/sgml/ref/create_subscription.sgml | 24 +-
src/backend/access/transam/xact.c | 13 +
src/backend/commands/subscriptioncmds.c | 66 +-
src/backend/postmaster/bgworker.c | 3 +
src/backend/replication/logical/Makefile | 1 +
.../replication/logical/applybgworker.c | 802 ++++++++++++++++++
src/backend/replication/logical/decode.c | 10 +-
src/backend/replication/logical/launcher.c | 130 ++-
src/backend/replication/logical/origin.c | 26 +-
src/backend/replication/logical/proto.c | 41 +-
.../replication/logical/reorderbuffer.c | 10 +-
src/backend/replication/logical/tablesync.c | 10 +-
src/backend/replication/logical/worker.c | 692 +++++++++++----
src/backend/replication/pgoutput/pgoutput.c | 6 +-
src/backend/utils/activity/wait_event.c | 3 +
src/backend/utils/misc/guc.c | 12 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/bin/pg_dump/pg_dump.c | 6 +-
src/include/catalog/pg_subscription.h | 21 +-
src/include/replication/logicallauncher.h | 1 +
src/include/replication/logicalproto.h | 27 +-
src/include/replication/logicalworker.h | 1 +
src/include/replication/origin.h | 2 +-
src/include/replication/reorderbuffer.h | 7 +-
src/include/replication/worker_internal.h | 102 ++-
src/include/utils/wait_event.h | 1 +
src/test/regress/expected/subscription.out | 2 +-
src/tools/pgindent/typedefs.list | 5 +
32 files changed, 1869 insertions(+), 220 deletions(-)
create mode 100644 src/backend/replication/logical/applybgworker.c
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 670a5406d6..13c9e8eca1 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7873,11 +7873,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry role="catalog_table_entry"><para role="column_definition">
- <structfield>substream</structfield> <type>bool</type>
+ <structfield>substream</structfield> <type>char</type>
</para>
<para>
- If true, the subscription will allow streaming of in-progress
- transactions
+ Controls how to handle the streaming of in-progress transactions:
+ <literal>f</literal> = disallow streaming of in-progress transactions,
+ <literal>t</literal> = spill the changes of in-progress transactions to
+ disk and apply at once after the transaction is committed on the
+ publisher,
+ <literal>p</literal> = apply changes directly using a background worker
</para></entry>
</row>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..6bbd986195 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4970,6 +4970,31 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
</listitem>
</varlistentry>
+ <varlistentry id="guc-max-apply-bgworkers-per-subscription" xreflabel="max_apply_bgworkers_per_subscription">
+ <term><varname>max_apply_bgworkers_per_subscription</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>max_apply_bgworkers_per_subscription</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Maximum number of apply background workers per subscription. This
+ parameter controls the amount of parallelism of the streaming of
+ in-progress transactions when subscription parameter
+ <literal>streaming = parallel</literal>.
+ </para>
+ <para>
+ The apply background workers are taken from the pool defined by
+ <varname>max_logical_replication_workers</varname>.
+ </para>
+ <para>
+ The default value is 2. This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command
+ line.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</sect2>
diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml
index bdf1e7b727..92997f9299 100644
--- a/doc/src/sgml/logical-replication.sgml
+++ b/doc/src/sgml/logical-replication.sgml
@@ -1153,6 +1153,16 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER
might not violate any constraint. This can easily make the subscriber
inconsistent.
</para>
+
+ <para>
+ When the streaming mode is <literal>parallel</literal>, the finish LSN of
+ failed transactions may not be logged. In that case, it may be necessary to
+ change the streaming mode to <literal>on</literal> and cause the same
+ conflicts again so the finish LSN of the failed transaction will be written
+ to the server log. For the usage of finish LSN, please refer to <link
+ linkend="sql-altersubscription"><command>ALTER SUBSCRIPTION ...
+ SKIP</command></link>.
+ </para>
</sect1>
<sect1 id="logical-replication-restrictions">
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..7e88ba9631 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -6809,6 +6809,25 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term>Int64 (XLogRecPtr)</term>
+ <listitem>
+ <para>
+ The LSN of the abort.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>Int64 (TimestampTz)</term>
+ <listitem>
+ <para>
+ Abort timestamp of the transaction. The value is in number
+ of microseconds since PostgreSQL epoch (2000-01-01).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term>Int32 (TransactionId)</term>
<listitem>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 34b3264b26..71dd4aca81 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -217,13 +217,29 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</varlistentry>
<varlistentry>
- <term><literal>streaming</literal> (<type>boolean</type>)</term>
+ <term><literal>streaming</literal> (<type>enum</type>)</term>
<listitem>
<para>
Specifies whether to enable streaming of in-progress transactions
- for this subscription. By default, all transactions
- are fully decoded on the publisher and only then sent to the
- subscriber as a whole.
+ for this subscription. The default value is <literal>off</literal>,
+ meaning all transactions are fully decoded on the publisher and only
+ then sent to the subscriber as a whole.
+ </para>
+
+ <para>
+ If set to <literal>on</literal>, the incoming changes are written to
+ temporary files and then applied only after the transaction is
+ committed on the publisher.
+ </para>
+
+ <para>
+ If set to <literal>parallel</literal>, incoming changes are directly
+ applied via one of the apply background workers, if available. If no
+ background worker is free to handle streaming transaction then the
+ changes are written to temporary files and applied after the
+ transaction is committed. Note that if an error happens when
+ applying changes in a background worker, the finish LSN of the
+ remote transaction might not be reported in the server log.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 116de1175b..3e61a57b50 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -1711,6 +1711,7 @@ RecordTransactionAbort(bool isSubXact)
int nchildren;
TransactionId *children;
TimestampTz xact_time;
+ bool replorigin;
/*
* If we haven't been assigned an XID, nobody will care whether we aborted
@@ -1741,6 +1742,13 @@ RecordTransactionAbort(bool isSubXact)
elog(PANIC, "cannot abort transaction %u, it was already committed",
xid);
+ /*
+ * Are we using the replication origins feature? Or, in other words,
+ * are we replaying remote actions?
+ */
+ replorigin = (replorigin_session_origin != InvalidRepOriginId &&
+ replorigin_session_origin != DoNotReplicateId);
+
/* Fetch the data we need for the abort record */
nrels = smgrGetPendingDeletes(false, &rels);
nchildren = xactGetCommittedChildren(&children);
@@ -1765,6 +1773,11 @@ RecordTransactionAbort(bool isSubXact)
MyXactFlags, InvalidTransactionId,
NULL);
+ if (replorigin)
+ /* Move LSNs forward for this replication origin */
+ replorigin_session_advance(replorigin_session_origin_lsn,
+ XactLastRecEnd);
+
/*
* Report the latest async abort LSN, so that the WAL writer knows to
* flush this abort. There's nothing to be gained by delaying this, since
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bdc1208724..d4b2616ee6 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -83,7 +83,7 @@ typedef struct SubOpts
bool copy_data;
bool refresh;
bool binary;
- bool streaming;
+ char streaming;
bool twophase;
bool disableonerr;
XLogRecPtr lsn;
@@ -95,6 +95,62 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub,
static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
+/*
+ * Extract the streaming mode value from a DefElem. This is like
+ * defGetBoolean() but also accepts the special value of "parallel".
+ */
+static char
+defGetStreamingMode(DefElem *def)
+{
+ /*
+ * If no parameter value given, assume "true" is meant.
+ */
+ if (def->arg == NULL)
+ return SUBSTREAM_ON;
+
+ /*
+ * Allow 0, 1, "false", "true", "off", "on" or "parallel".
+ */
+ switch (nodeTag(def->arg))
+ {
+ case T_Integer:
+ switch (intVal(def->arg))
+ {
+ case 0:
+ return SUBSTREAM_OFF;
+ case 1:
+ return SUBSTREAM_ON;
+ default:
+ /* otherwise, error out below */
+ break;
+ }
+ break;
+ default:
+ {
+ char *sval = defGetString(def);
+
+ /*
+ * The set of strings accepted here should match up with the
+ * grammar's opt_boolean_or_string production.
+ */
+ if (pg_strcasecmp(sval, "false") == 0 ||
+ pg_strcasecmp(sval, "off") == 0)
+ return SUBSTREAM_OFF;
+ if (pg_strcasecmp(sval, "true") == 0 ||
+ pg_strcasecmp(sval, "on") == 0)
+ return SUBSTREAM_ON;
+ if (pg_strcasecmp(sval, "parallel") == 0)
+ return SUBSTREAM_PARALLEL;
+ }
+ break;
+ }
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("%s requires a Boolean value or \"parallel\"",
+ def->defname)));
+ return SUBSTREAM_OFF; /* keep compiler quiet */
+}
+
/*
* Common option parsing function for CREATE and ALTER SUBSCRIPTION commands.
*
@@ -132,7 +188,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
if (IsSet(supported_opts, SUBOPT_BINARY))
opts->binary = false;
if (IsSet(supported_opts, SUBOPT_STREAMING))
- opts->streaming = false;
+ opts->streaming = SUBSTREAM_OFF;
if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
opts->twophase = false;
if (IsSet(supported_opts, SUBOPT_DISABLE_ON_ERR))
@@ -233,7 +289,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
errorConflictingDefElem(defel, pstate);
opts->specified_opts |= SUBOPT_STREAMING;
- opts->streaming = defGetBoolean(defel);
+ opts->streaming = defGetStreamingMode(defel);
}
else if (strcmp(defel->defname, "two_phase") == 0)
{
@@ -600,7 +656,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner);
values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled);
values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary);
- values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming);
+ values[Anum_pg_subscription_substream - 1] = CharGetDatum(opts.streaming);
values[Anum_pg_subscription_subtwophasestate - 1] =
CharGetDatum(opts.twophase ?
LOGICALREP_TWOPHASE_STATE_PENDING :
@@ -1059,7 +1115,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
if (IsSet(opts.specified_opts, SUBOPT_STREAMING))
{
values[Anum_pg_subscription_substream - 1] =
- BoolGetDatum(opts.streaming);
+ CharGetDatum(opts.streaming);
replaces[Anum_pg_subscription_substream - 1] = true;
}
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 40601aefd9..40ccb8993c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -128,6 +128,9 @@ static const struct
},
{
"ApplyWorkerMain", ApplyWorkerMain
+ },
+ {
+ "ApplyBgworkerMain", ApplyBgworkerMain
}
};
diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile
index c4e2fdeb71..cbfb5d794e 100644
--- a/src/backend/replication/logical/Makefile
+++ b/src/backend/replication/logical/Makefile
@@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(srcdir) $(CPPFLAGS)
OBJS = \
+ applybgworker.o \
decode.o \
launcher.o \
logical.o \
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
new file mode 100644
index 0000000000..aa222490a0
--- /dev/null
+++ b/src/backend/replication/logical/applybgworker.c
@@ -0,0 +1,802 @@
+/*-------------------------------------------------------------------------
+ * applybgworker.c
+ * Support routines for applying xact by apply background worker
+ *
+ * Copyright (c) 2016-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/logical/applybgworker.c
+ *
+ * This file contains routines that are intended to support setting up, using,
+ * and tearing down a ApplyBgworkerState.
+ *
+ * Refer to the comments in file header of logical/worker.c to see more
+ * information about apply background worker.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "libpq/pqformat.h"
+#include "mb/pg_wchar.h"
+#include "pgstat.h"
+#include "postmaster/interrupt.h"
+#include "replication/logicallauncher.h"
+#include "replication/logicalworker.h"
+#include "replication/origin.h"
+#include "replication/walreceiver.h"
+#include "replication/worker_internal.h"
+#include "storage/ipc.h"
+#include "storage/procarray.h"
+#include "tcop/tcopprot.h"
+#include "utils/inval.h"
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+#include "utils/syscache.h"
+
+#define PG_LOGICAL_APPLY_SHM_MAGIC 0x79fb2447 // TODO Consider change
+
+/*
+ * DSM keys for apply background worker. Unlike other parallel execution code,
+ * since we don't need to worry about DSM keys conflicting with plan_node_id we
+ * can use small integers.
+ */
+#define APPLY_BGWORKER_KEY_SHARED 1
+#define APPLY_BGWORKER_KEY_MQ 2
+
+/* Queue size of DSM, 16 MB for now. */
+#define DSM_QUEUE_SIZE 160000000
+
+/*
+ * There are three fields in message: start_lsn, end_lsn and send_time. Because
+ * we have updated these statistics in apply worker, we could ignore these
+ * fields in apply background worker. (see function LogicalRepApplyLoop)
+ */
+#define IGNORE_SIZE_IN_MESSAGE (3 * sizeof(uint64))
+
+/*
+ * Entry for a hash table we use to map from xid to our apply background worker
+ * state.
+ */
+typedef struct ApplyBgworkerEntry
+{
+ TransactionId xid;
+ ApplyBgworkerState *wstate;
+} ApplyBgworkerEntry;
+
+/* Apply background workers hash table (initialized on first use) */
+static HTAB *ApplyWorkersHash = NULL;
+static List *ApplyWorkersFreeList = NIL;
+static List *ApplyWorkersList = NIL;
+
+/*
+ * Information shared between main apply worker and apply background worker.
+ */
+volatile ApplyBgworkerShared *MyParallelShared = NULL;
+
+List *subxactlist = NIL;
+
+static bool apply_bgworker_can_start(TransactionId xid);
+static ApplyBgworkerState *apply_bgworker_setup(void);
+static void apply_bgworker_setup_dsm(ApplyBgworkerState *wstate);
+
+/*
+ * Check if starting a new apply background worker is allowed.
+ */
+static bool
+apply_bgworker_can_start(TransactionId xid)
+{
+ if (!TransactionIdIsValid(xid))
+ return false;
+
+ /*
+ * Don't start a new background worker if not in streaming parallel mode.
+ */
+ if (MySubscription->stream != SUBSTREAM_PARALLEL)
+ return false;
+
+ /*
+ * Don't start a new background worker if user has set skiplsn as it's
+ * possible that user want to skip the streaming transaction. For
+ * streaming transaction, we need to spill the transaction to disk so that
+ * we can get the last LSN of the transaction to judge whether to skip
+ * before starting to apply the change.
+ */
+ if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
+ return false;
+
+ /*
+ * For streaming transactions that are being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time. So, we don't start new apply
+ * background worker in this case.
+ */
+ if (!AllTablesyncsReady())
+ return false;
+
+ return true;
+}
+
+/*
+ * Try to start an apply background worker and, if successful, cache it in
+ * ApplyWorkersHash keyed by the specified xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_start(TransactionId xid)
+{
+ bool found;
+ int server_version;
+ ApplyBgworkerState *wstate;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!apply_bgworker_can_start(xid))
+ return NULL;
+
+ /* First time through, initialize apply workers hashtable */
+ if (ApplyWorkersHash == NULL)
+ {
+ HASHCTL ctl;
+
+ MemSet(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(TransactionId);
+ ctl.entrysize = sizeof(ApplyBgworkerEntry);
+ ctl.hcxt = ApplyContext;
+
+ ApplyWorkersHash = hash_create("logical apply workers hash", 8, &ctl,
+ HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+ }
+
+ /*
+ * Now, we try to get an apply background worker. If there is at least one
+ * worker in the free list, then take one. Otherwise, we try to start a
+ * new apply background worker.
+ */
+ if (list_length(ApplyWorkersFreeList) > 0)
+ {
+ wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
+ ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
+ Assert(wstate->shared->status == APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ wstate = apply_bgworker_setup();
+
+ if (wstate == NULL)
+ return NULL;
+ }
+
+ /*
+ * Create entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_ENTER, &found);
+ if (found)
+ elog(ERROR, "hash table corrupted");
+
+ /* Fill up the hash entry */
+ wstate->shared->status = APPLY_BGWORKER_BUSY;
+
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ wstate->shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
+
+ wstate->shared->stream_xid = xid;
+ entry->wstate = wstate;
+ entry->xid = xid;
+
+ return wstate;
+}
+
+/*
+ * Try to look up worker inside ApplyWorkersHash for requested xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_find(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!TransactionIdIsValid(xid))
+ return NULL;
+
+ if (ApplyWorkersHash == NULL)
+ return NULL;
+
+ /*
+ * Find entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
+ if (found)
+ {
+ char status = entry->wstate->shared->status;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ entry->wstate->shared->n,
+ entry->wstate->shared->stream_xid)));
+
+ Assert(status == APPLY_BGWORKER_BUSY);
+
+ return entry->wstate;
+ }
+ else
+ return NULL;
+}
+
+/*
+ * Add the worker to the free list and remove the entry from the hash table.
+ */
+void
+apply_bgworker_free(ApplyBgworkerState *wstate)
+{
+ MemoryContext oldctx;
+ TransactionId xid = wstate->shared->stream_xid;
+
+ Assert(wstate->shared->status == APPLY_BGWORKER_FINISHED);
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ hash_search(ApplyWorkersHash, &xid, HASH_REMOVE, NULL);
+
+ elog(DEBUG1, "adding finished apply worker #%u for xid %u to the free list",
+ wstate->shared->n, wstate->shared->stream_xid);
+
+ ApplyWorkersFreeList = lappend(ApplyWorkersFreeList, wstate);
+
+ MemoryContextSwitchTo(oldctx);
+}
+
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *shared)
+{
+ shm_mq_result shmq_res;
+ PGPROC *registrant;
+ ErrorContextCallback errcallback;
+
+ registrant = BackendPidGetProc(MyBgworkerEntry->bgw_notify_pid);
+ SetLatch(®istrant->procLatch);
+
+ /*
+ * Push apply error context callback. Fields will be filled applying a
+ * change.
+ */
+ errcallback.callback = apply_error_callback;
+ errcallback.previous = error_context_stack;
+ error_context_stack = &errcallback;
+
+ for (;;)
+ {
+ void *data;
+ Size len;
+ int c;
+ StringInfoData s;
+ MemoryContext oldctx;
+
+ CHECK_FOR_INTERRUPTS();
+
+ /* Ensure we are reading the data into our memory context. */
+ oldctx = MemoryContextSwitchTo(ApplyMessageContext);
+
+ shmq_res = shm_mq_receive(mqh, &len, &data, false);
+
+ if (shmq_res != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("lost connection to the main apply worker")));
+
+ if (len == 0)
+ break;
+
+ s.cursor = 0;
+ s.maxlen = -1;
+ s.data = (char *) data;
+ s.len = len;
+
+ /*
+ * We use first byte of message for additional communication between
+ * main Logical replication worker and apply background workers, so if
+ * it differs from 'w', then process it first.
+ */
+ c = pq_getmsgbyte(&s);
+ switch (c)
+ {
+ /* End message of streaming chunk */
+ case LOGICAL_REP_MSG_STREAM_STOP:
+ elog(DEBUG1, "[Apply BGW #%u] ended processing streaming chunk,"
+ "waiting on shm_mq_receive", shared->n);
+
+ in_streamed_transaction = false;
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ continue;
+ case 'w':
+ break;
+ default:
+ elog(ERROR, "[Apply BGW #%u] unexpected message \"%c\"",
+ shared->n, c);
+ break;
+ }
+
+ /* Ignore statistics fields that have been updated. */
+ s.cursor += IGNORE_SIZE_IN_MESSAGE;
+
+ apply_dispatch(&s);
+
+ if (ConfigReloadPending)
+ {
+ ConfigReloadPending = false;
+ ProcessConfigFile(PGC_SIGHUP);
+ }
+
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
+ }
+
+ MemoryContextSwitchTo(TopMemoryContext);
+ MemoryContextReset(ApplyContext);
+
+ /* Pop the error context stack */
+ error_context_stack = errcallback.previous;
+
+ elog(DEBUG1, "[Apply BGW #%u] exiting", shared->n);
+
+ /* Signal main process that we are done. */
+ SetLatch(®istrant->procLatch);
+}
+
+/*
+ * Set the exit status so that the main apply worker can realize we have
+ * shutdown.
+ */
+static void
+apply_bgworker_shutdown(int code, Datum arg)
+{
+ SpinLockAcquire(&MyParallelShared->mutex);
+ MyParallelShared->status = APPLY_BGWORKER_EXIT;
+ SpinLockRelease(&MyParallelShared->mutex);
+
+ dsm_detach((dsm_segment *) DatumGetPointer(arg));
+}
+
+/*
+ * Apply Background Worker entry point
+ */
+void
+ApplyBgworkerMain(Datum main_arg)
+{
+ volatile ApplyBgworkerShared *shared;
+
+ dsm_handle handle;
+ dsm_segment *seg;
+ shm_toc *toc;
+ shm_mq *mq;
+ shm_mq_handle *mqh;
+ MemoryContext oldcontext;
+ RepOriginId originid;
+ int worker_slot = DatumGetInt32(main_arg);
+ char originname[NAMEDATALEN];
+
+ MemoryContextSwitchTo(TopMemoryContext);
+
+ /* Init the memory context for the apply background worker to work in. */
+ ApplyContext = AllocSetContextCreate(TopMemoryContext,
+ "ApplyContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Init the ApplyMessageContext which we clean up after each replication
+ * protocol message.
+ */
+ ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+ "ApplyMessageContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /* Setup signal handling */
+ pqsignal(SIGHUP, SignalHandlerForConfigReload);
+ pqsignal(SIGTERM, die);
+ BackgroundWorkerUnblockSignals();
+
+ /*
+ * Connect to the dynamic shared memory segment.
+ *
+ * The backend that registered this worker passed us the ID of a shared
+ * memory segment to which we must attach for further instructions. In
+ * order to attach to dynamic shared memory, we need a resource owner.
+ * Once we've mapped the segment in our address space, attach to the table
+ * of contents so we can locate the various data structures we'll need to
+ * find within the segment.
+ */
+ CurrentResourceOwner = ResourceOwnerCreate(NULL, "Logical apply worker");
+ memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsm_handle));
+ seg = dsm_attach(handle);
+ if (seg == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("unable to map dynamic shared memory segment")));
+ toc = shm_toc_attach(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg));
+ if (toc == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("bad magic number in dynamic shared memory segment")));
+
+ before_shmem_exit(apply_bgworker_shutdown, PointerGetDatum(seg));
+
+ /* Look up the shared information. */
+ shared = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_SHARED, false);
+ MyParallelShared = shared;
+
+ /*
+ * Attach to the message queue.
+ */
+ mq = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_MQ, false);
+ shm_mq_set_receiver(mq, MyProc);
+ mqh = shm_mq_attach(mq, seg, NULL);
+
+ /* Run as replica session replication role. */
+ SetConfigOption("session_replication_role", "replica",
+ PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Now, we have initialized DSM. Attach to slot.
+ */
+ logicalrep_worker_attach(worker_slot);
+
+ /* Connect to our database. */
+ BackgroundWorkerInitializeConnectionByOid(MyLogicalRepWorker->dbid,
+ MyLogicalRepWorker->userid,
+ 0);
+
+ /*
+ * Set always-secure search path, so malicious users can't redirect user
+ * code (e.g. pg_index.indexprs).
+ */
+ SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Set the client encoding to the database encoding, since that is what
+ * the leader will expect.
+ */
+ SetClientEncoding(GetDatabaseEncoding());
+
+ stream_xid = shared->stream_xid;
+
+ StartTransactionCommand();
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ MySubscription = GetSubscription(MyLogicalRepWorker->subid, true);
+ if (!MySubscription)
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply worker for subscription %u will not "
+ "start because the subscription was removed during startup",
+ MyLogicalRepWorker->subid)));
+ proc_exit(0);
+ }
+
+ MySubscriptionValid = true;
+ MemoryContextSwitchTo(oldcontext);
+
+ /* Setup synchronous commit according to the user's wishes */
+ SetConfigOption("synchronous_commit", MySubscription->synccommit,
+ PGC_BACKEND, PGC_S_OVERRIDE);
+
+ /* Keep us informed about subscription changes. */
+ CacheRegisterSyscacheCallback(SUBSCRIPTIONOID,
+ subscription_change_cb,
+ (Datum) 0);
+
+ CommitTransactionCommand();
+
+ /* Setup replication origin tracking. */
+ StartTransactionCommand();
+ snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
+ originid = replorigin_by_name(originname, true);
+ if (!OidIsValid(originid))
+ originid = replorigin_create(originname);
+
+ /*
+ * The apply background worker doesn't need to monopolize this replication
+ * origin which was already acquired by its leader process.
+ */
+ replorigin_session_setup(originid, false);
+ replorigin_session_origin = originid;
+ CommitTransactionCommand();
+
+ /*
+ * Allocate the origin name in long-lived context for error context
+ * message.
+ */
+ apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+ originname);
+
+ elog(DEBUG1, "[Apply BGW #%u] started", shared->n);
+
+ LogicalApplyBgwLoop(mqh, shared);
+
+ /*
+ * We're done. Explicitly detach the shared memory segment so that we
+ * don't get a resource leak warning at commit time. This will fire any
+ * on_dsm_detach callbacks we've registered, as well. Once that's done,
+ * we can go ahead and exit.
+ */
+ dsm_detach(seg);
+ proc_exit(0);
+}
+
+/*
+ * Set up a dynamic shared memory segment.
+ *
+ * We set up a control region that contains a ApplyBgworkerShared,
+ * plus one region per message queue. There are as many message queues as
+ * the number of workers.
+ */
+static void
+apply_bgworker_setup_dsm(ApplyBgworkerState *wstate)
+{
+ shm_toc_estimator e;
+ Size segsize;
+ dsm_segment *seg;
+ shm_toc *toc;
+ ApplyBgworkerShared *shared;
+ shm_mq *mq;
+ int64 queue_size = DSM_QUEUE_SIZE;
+ int server_version;
+
+ /*
+ * Estimate how much shared memory we need.
+ *
+ * Because the TOC machinery may choose to insert padding of oddly-sized
+ * requests, we must estimate each chunk separately.
+ *
+ * We need one key to register the location of the header, and we need
+ * another key to track the location of the message queue.
+ */
+ shm_toc_initialize_estimator(&e);
+ shm_toc_estimate_chunk(&e, sizeof(ApplyBgworkerShared));
+ shm_toc_estimate_chunk(&e, (Size) queue_size);
+
+ shm_toc_estimate_keys(&e, 1 + 1);
+ segsize = shm_toc_estimate(&e);
+
+ /* Create the shared memory segment and establish a table of contents. */
+ seg = dsm_create(shm_toc_estimate(&e), 0);
+ toc = shm_toc_create(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg),
+ segsize);
+
+ /* Set up the header region. */
+ shared = shm_toc_allocate(toc, sizeof(ApplyBgworkerShared));
+ SpinLockInit(&shared->mutex);
+ shared->status = APPLY_BGWORKER_BUSY;
+
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
+
+ shared->stream_xid = stream_xid;
+ shared->n = list_length(ApplyWorkersList) + 1;
+
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_SHARED, shared);
+
+ /* Set up message queue for the worker. */
+ mq = shm_mq_create(shm_toc_allocate(toc, (Size) queue_size),
+ (Size) queue_size);
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_MQ, mq);
+ shm_mq_set_sender(mq, MyProc);
+
+ /* Attach the queue. */
+ wstate->mq_handle = shm_mq_attach(mq, seg, NULL);
+
+ /* Return results to caller. */
+ wstate->dsm_seg = seg;
+ wstate->shared = shared;
+}
+
+/*
+ * Start apply background worker process and allocate shared memory for it.
+ */
+static ApplyBgworkerState *
+apply_bgworker_setup(void)
+{
+ MemoryContext oldcontext;
+ bool launched;
+ ApplyBgworkerState *wstate;
+ int napplyworkers;
+
+ elog(DEBUG1, "setting up apply worker #%u", list_length(ApplyWorkersList) + 1);
+
+ /* Check if there are free worker slot(s) */
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+ napplyworkers = logicalrep_apply_bgworker_count(MyLogicalRepWorker->subid);
+ LWLockRelease(LogicalRepWorkerLock);
+ if (napplyworkers >= max_apply_bgworkers_per_subscription)
+ return NULL;
+
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ wstate = (ApplyBgworkerState *) palloc0(sizeof(ApplyBgworkerState));
+
+ /* Setup shared memory */
+ apply_bgworker_setup_dsm(wstate);
+
+ launched = logicalrep_worker_launch(MyLogicalRepWorker->dbid,
+ MySubscription->oid,
+ MySubscription->name,
+ MyLogicalRepWorker->userid,
+ InvalidOid,
+ dsm_segment_handle(wstate->dsm_seg));
+
+ if (launched)
+ ApplyWorkersList = lappend(ApplyWorkersList, wstate);
+ else
+ {
+ dsm_detach(wstate->dsm_seg);
+ wstate->dsm_seg = NULL;
+
+ pfree(wstate);
+ wstate = NULL;
+ }
+
+ MemoryContextSwitchTo(oldcontext);
+
+ return wstate;
+}
+
+/*
+ * Send the data to the specified apply background worker via shared-memory queue.
+ */
+void
+apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes, const void *data)
+{
+ shm_mq_result result;
+
+ result = shm_mq_send(wstate->mq_handle, nbytes, data, false, true);
+
+ if (result != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("could not send tuples to shared-memory queue")));
+}
+
+/*
+ * Wait until the status of apply background worker reaches the
+ * 'wait_for_status'
+ */
+void
+apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status)
+{
+ for (;;)
+ {
+ char status;
+
+ SpinLockAcquire(&wstate->shared->mutex);
+ status = wstate->shared->status;
+ SpinLockRelease(&wstate->shared->mutex);
+
+ /* Done if already in correct status. */
+ if (status == wait_for_status)
+ break;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ wstate->shared->n, wstate->shared->stream_xid)));
+
+ /* Wait to be signalled. */
+ WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 0,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE);
+
+ /* Reset the latch so we don't spin. */
+ ResetLatch(MyLatch);
+
+ /* An interrupt may have occurred while we were waiting. */
+ CHECK_FOR_INTERRUPTS();
+ }
+}
+
+/*
+ * Check the status of workers and report an error if any apply background
+ * worker has exited unexpectedly.
+ */
+void
+apply_bgworker_check_status(void)
+{
+ ListCell *lc;
+
+ if (am_apply_bgworker() || MySubscription->stream != SUBSTREAM_PARALLEL)
+ return;
+
+ foreach(lc, ApplyWorkersList)
+ {
+ ApplyBgworkerState *wstate = (ApplyBgworkerState *) lfirst(lc);
+
+ /*
+ * We don't lock here as in the worst case we will just detect the
+ * failure of worker a bit later.
+ */
+ if (wstate->shared->status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u exited unexpectedly",
+ wstate->shared->n)));
+ }
+
+ /*
+ * Exit if any relation is not in the READY state and if any worker is
+ * handling the streaming transaction at the same time. Because for
+ * streaming transactions that is being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time.
+ */
+ if (list_length(ApplyWorkersFreeList) != list_length(ApplyWorkersList) &&
+ !AllTablesyncsReady())
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply workers for subscription \"%s\" will restart",
+ MySubscription->name),
+ errdetail("Cannot handle streamed replication transaction by apply "
+ "background workers until all tables are synchronized")));
+
+ proc_exit(0);
+ }
+}
+
+/* Set the apply background worker status */
+void
+apply_bgworker_set_status(ApplyBgworkerStatus status)
+{
+ if (!am_apply_bgworker())
+ return;
+
+ elog(DEBUG1, "[Apply BGW #%u] set status to %d", MyParallelShared->n, status);
+
+ SpinLockAcquire(&MyParallelShared->mutex);
+ MyParallelShared->status = status;
+ SpinLockRelease(&MyParallelShared->mutex);
+}
+
+/*
+ * Define a savepoint for a subxact in apply background worker if needed.
+ *
+ * Inside apply background worker we can figure out that new subtransaction was
+ * started if new change arrived with different xid. In that case we can define
+ * named savepoint, so that we were able to commit/rollback it separately
+ * later.
+ * Special case is if the first change comes from subtransaction, then
+ * we check that current_xid differs from stream_xid.
+ */
+void
+apply_bgworker_subxact_info_add(TransactionId current_xid)
+{
+ if (current_xid != stream_xid &&
+ !list_member_int(subxactlist, (int) current_xid))
+ {
+ MemoryContext oldctx;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", current_xid);
+
+ elog(DEBUG1, "[Apply BGW #%u] defining savepoint %s",
+ MyParallelShared->n, spname);
+
+ DefineSavepoint(spname);
+ CommitTransactionCommand();
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+ subxactlist = lappend_int(subxactlist, (int) current_xid);
+ MemoryContextSwitchTo(oldctx);
+ }
+}
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c5c6a2ba68..d4d5093a0b 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -651,9 +651,10 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
{
for (i = 0; i < parsed->nsubxacts; i++)
{
- ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr);
+ ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr,
+ commit_time);
}
- ReorderBufferForget(ctx->reorder, xid, buf->origptr);
+ ReorderBufferForget(ctx->reorder, xid, buf->origptr, commit_time);
return;
}
@@ -821,10 +822,11 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
for (i = 0; i < parsed->nsubxacts; i++)
{
ReorderBufferAbort(ctx->reorder, parsed->subxacts[i],
- buf->record->EndRecPtr);
+ buf->record->EndRecPtr, abort_time);
}
- ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr);
+ ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr,
+ abort_time);
}
/* update the decoding stats */
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 3bbd522724..d92bfaf6d6 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -54,6 +54,7 @@
int max_logical_replication_workers = 4;
int max_sync_workers_per_subscription = 2;
+int max_apply_bgworkers_per_subscription = 2;
LogicalRepWorker *MyLogicalRepWorker = NULL;
@@ -73,6 +74,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg);
static void logicalrep_worker_onexit(int code, Datum arg);
static void logicalrep_worker_detach(void);
static void logicalrep_worker_cleanup(LogicalRepWorker *worker);
+static void logicalrep_worker_stop_internal(LogicalRepWorker *worker);
static bool on_commit_launcher_wakeup = false;
@@ -151,8 +153,10 @@ get_subscription_list(void)
*
* This is only needed for cleaning up the shared memory in case the worker
* fails to attach.
+ *
+ * Return false if the attach fails. Otherwise return true.
*/
-static void
+static bool
WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
uint16 generation,
BackgroundWorkerHandle *handle)
@@ -168,11 +172,11 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
- /* Worker either died or has started; no need to do anything. */
+ /* Worker either died or has started. Return false if died. */
if (!worker->in_use || worker->proc)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return worker->in_use;
}
LWLockRelease(LogicalRepWorkerLock);
@@ -187,7 +191,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
if (generation == worker->generation)
logicalrep_worker_cleanup(worker);
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -223,6 +227,13 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running)
{
LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+ /*
+ * We are only interested in the main apply worker or table sync worker
+ * here.
+ */
+ if (w->subworker)
+ continue;
+
if (w->in_use && w->subid == subid && w->relid == relid &&
(!only_running || w->proc))
{
@@ -259,11 +270,11 @@ logicalrep_workers_find(Oid subid, bool only_running)
}
/*
- * Start new apply background worker, if possible.
+ * Start new background worker, if possible.
*/
-void
+bool
logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
- Oid relid)
+ Oid relid, dsm_handle subworker_dsm)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
@@ -273,6 +284,10 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
LogicalRepWorker *worker = NULL;
int nsyncworkers;
TimestampTz now;
+ bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
+
+ /* Sanity check : we don't support table sync in subworker. */
+ Assert(!(is_subworker && OidIsValid(relid)));
ereport(DEBUG1,
(errmsg_internal("starting logical replication worker for subscription \"%s\"",
@@ -350,7 +365,7 @@ retry:
if (OidIsValid(relid) && nsyncworkers >= max_sync_workers_per_subscription)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -364,7 +379,7 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of logical replication worker slots"),
errhint("You might need to increase max_logical_replication_workers.")));
- return;
+ return false;
}
/* Prepare the worker slot. */
@@ -379,6 +394,7 @@ retry:
worker->relstate = SUBREL_STATE_UNKNOWN;
worker->relstate_lsn = InvalidXLogRecPtr;
worker->stream_fileset = NULL;
+ worker->subworker = is_subworker;
worker->last_lsn = InvalidXLogRecPtr;
TIMESTAMP_NOBEGIN(worker->last_send_time);
TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -396,19 +412,31 @@ retry:
BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
- snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
+ if (is_subworker)
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyBgworkerMain");
+ else
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
if (OidIsValid(relid))
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u sync %u", subid, relid);
+ else if (is_subworker)
+ snprintf(bgw.bgw_name, BGW_MAXLEN,
+ "logical replication apply background worker for subscription %u", subid);
else
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u", subid);
+
snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = Int32GetDatum(slot);
+ if (is_subworker)
+ memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle));
+
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
/* Failed to start worker, so clean up the worker slot. */
@@ -421,11 +449,11 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of background worker slots"),
errhint("You might need to increase max_worker_processes.")));
- return;
+ return false;
}
/* Now wait until it attaches. */
- WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
+ return WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
}
/*
@@ -436,18 +464,27 @@ void
logicalrep_worker_stop(Oid subid, Oid relid)
{
LogicalRepWorker *worker;
- uint16 generation;
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
worker = logicalrep_worker_find(subid, relid, false);
- /* No worker, nothing to do. */
- if (!worker)
- {
- LWLockRelease(LogicalRepWorkerLock);
- return;
- }
+ if (worker)
+ logicalrep_worker_stop_internal(worker);
+
+ LWLockRelease(LogicalRepWorkerLock);
+}
+
+/*
+ * Workhorse for logicalrep_worker_stop() and logicalrep_worker_detach(). Stop
+ * the worker and wait for it to die.
+ */
+static void
+logicalrep_worker_stop_internal(LogicalRepWorker *worker)
+{
+ uint16 generation;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
/*
* Remember which generation was our worker so we can check if what we see
@@ -485,10 +522,7 @@ logicalrep_worker_stop(Oid subid, Oid relid)
* different, meaning that a different worker has taken the slot.
*/
if (!worker->in_use || worker->generation != generation)
- {
- LWLockRelease(LogicalRepWorkerLock);
return;
- }
/* Worker has assigned proc, so it has started. */
if (worker->proc)
@@ -522,8 +556,6 @@ logicalrep_worker_stop(Oid subid, Oid relid)
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
}
-
- LWLockRelease(LogicalRepWorkerLock);
}
/*
@@ -599,6 +631,29 @@ logicalrep_worker_attach(int slot)
static void
logicalrep_worker_detach(void)
{
+ /*
+ * If we are the main apply worker, stop all the apply background workers
+ * we started before.
+ */
+ if (!MyLogicalRepWorker->subworker)
+ {
+ List *workers;
+ ListCell *lc;
+
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+
+ workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true);
+ foreach(lc, workers)
+ {
+ LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc);
+
+ if (w->subworker)
+ logicalrep_worker_stop_internal(w);
+ }
+
+ LWLockRelease(LogicalRepWorkerLock);
+ }
+
/* Block concurrent access. */
LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
@@ -621,6 +676,7 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker)
worker->userid = InvalidOid;
worker->subid = InvalidOid;
worker->relid = InvalidOid;
+ worker->subworker = false;
}
/*
@@ -679,6 +735,30 @@ logicalrep_sync_worker_count(Oid subid)
return res;
}
+/*
+ * Count the number of registered (not necessarily running) apply background
+ * workers for a subscription.
+ */
+int
+logicalrep_apply_bgworker_count(Oid subid)
+{
+ int i;
+ int res = 0;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
+
+ /* Search for attached worker for a given subscription id. */
+ for (i = 0; i < max_logical_replication_workers; i++)
+ {
+ LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+
+ if (w->subid == subid && w->subworker)
+ res++;
+ }
+
+ return res;
+}
+
/*
* ApplyLauncherShmemSize
* Compute space needed for replication launcher shared memory
@@ -868,7 +948,7 @@ ApplyLauncherMain(Datum main_arg)
wait_time = wal_retrieve_retry_interval;
logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
- sub->owner, InvalidOid);
+ sub->owner, InvalidOid, DSM_HANDLE_INVALID);
}
}
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index 21937ab2d3..50c567fb6e 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1063,12 +1063,21 @@ ReplicationOriginExitCleanup(int code, Datum arg)
* array doesn't have to be searched when calling
* replorigin_session_advance().
*
- * Obviously only one such cached origin can exist per process and the current
+ * Normally only one such cached origin can exist per process and the current
* cached value can only be set again after the previous value is torn down
* with replorigin_session_reset().
+ *
+ * However, if the function parameter 'must_acquire' is false, we allow the
+ * process to use the same slot already acquired by another process. It's safe
+ * because 1) The only caller (apply background workers) will maintain the
+ * commit order by allowing only one process to commit at a time, so no two
+ * workers will be operating on the same origin at the same time (see comments
+ * in logical/worker.c). 2) Even though we try to advance the session's origin
+ * concurrently, it's safe to do so as we change/advance the session_origin
+ * LSNs under replicate_state LWLock.
*/
void
-replorigin_session_setup(RepOriginId node)
+replorigin_session_setup(RepOriginId node, bool must_acquire)
{
static bool registered_cleanup;
int i;
@@ -1110,7 +1119,7 @@ replorigin_session_setup(RepOriginId node)
if (curstate->roident != node)
continue;
- else if (curstate->acquired_by != 0)
+ else if (curstate->acquired_by != 0 && must_acquire)
{
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
@@ -1141,7 +1150,14 @@ replorigin_session_setup(RepOriginId node)
Assert(session_replication_state->roident != InvalidRepOriginId);
- session_replication_state->acquired_by = MyProcPid;
+ if (must_acquire)
+ session_replication_state->acquired_by = MyProcPid;
+ else if (session_replication_state->acquired_by == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+ errmsg("apply background worker could not find replication state slot for replication origin with OID %u",
+ node),
+ errdetail("There is no replication state slot set by its main apply worker.")));
LWLockRelease(ReplicationOriginLock);
@@ -1321,7 +1337,7 @@ pg_replication_origin_session_setup(PG_FUNCTION_ARGS)
name = text_to_cstring((text *) DatumGetPointer(PG_GETARG_DATUM(0)));
origin = replorigin_by_name(name, false);
- replorigin_session_setup(origin);
+ replorigin_session_setup(origin, true);
replorigin_session_origin = origin;
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index ff8513e2d2..47bd811fb7 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -1163,31 +1163,56 @@ logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data)
/*
* Write STREAM ABORT to the output stream. Note that xid and subxid will be
* same for the top-level transaction abort.
+ *
+ * If write_abort_lsn is true, send the abort_lsn and abort_time fields,
+ * otherwise don't.
*/
void
logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid)
+ ReorderBufferTXN *txn, XLogRecPtr abort_lsn,
+ bool write_abort_lsn)
{
pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT);
- Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid));
+ Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(txn->xid));
/* transaction ID */
pq_sendint32(out, xid);
- pq_sendint32(out, subxid);
+ pq_sendint32(out, txn->xid);
+
+ if (write_abort_lsn)
+ {
+ pq_sendint64(out, abort_lsn);
+ pq_sendint64(out, txn->xact_time.abort_time);
+ }
}
/*
* Read STREAM ABORT from the output stream.
+ *
+ * If read_abort_lsn is true, try to read the abort_lsn and abort_time fields,
+ * otherwise don't.
*/
void
-logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid)
+logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn)
{
- Assert(xid && subxid);
+ Assert(abort_data);
- *xid = pq_getmsgint(in, 4);
- *subxid = pq_getmsgint(in, 4);
+ abort_data->xid = pq_getmsgint(in, 4);
+ abort_data->subxid = pq_getmsgint(in, 4);
+
+ if (read_abort_lsn)
+ {
+ abort_data->abort_lsn = pq_getmsgint64(in);
+ abort_data->abort_time = pq_getmsgint64(in);
+ }
+ else
+ {
+ abort_data->abort_lsn = InvalidXLogRecPtr;
+ abort_data->abort_time = 0;
+ }
}
/*
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 88a37fde72..8989328046 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -2826,7 +2826,8 @@ ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
* disk.
*/
void
-ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2837,6 +2838,8 @@ ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
{
@@ -2911,7 +2914,8 @@ ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid)
* to this xid might re-create the transaction incompletely.
*/
void
-ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2922,6 +2926,8 @@ ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
rb->stream_abort(rb, txn, lsn);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 670c6fcada..8ffba7e2e5 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -568,7 +568,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
MySubscription->oid,
MySubscription->name,
MyLogicalRepWorker->userid,
- rstate->relid);
+ rstate->relid,
+ DSM_HANDLE_INVALID);
hentry->last_start_time = now;
}
}
@@ -589,6 +590,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
void
process_syncing_tables(XLogRecPtr current_lsn)
{
+ if (MyLogicalRepWorker->subworker)
+ return;
+
if (am_tablesync_worker())
process_syncing_tables_for_sync(current_lsn);
else
@@ -1273,7 +1277,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
* time this tablesync was launched.
*/
originid = replorigin_by_name(originname, false);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
*origin_startpos = replorigin_session_get_progress(false);
@@ -1384,7 +1388,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
true /* go backward */ , true /* WAL log */ );
UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
}
else
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 38e3b1c1b3..b5aae0e19a 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -22,8 +22,28 @@
* STREAMED TRANSACTIONS
* ---------------------
* Streamed transactions (large transactions exceeding a memory limit on the
- * upstream) are not applied immediately, but instead, the data is written
- * to temporary files and then applied at once when the final commit arrives.
+ * upstream) are applied using one of two approaches.
+ *
+ * 1) Separate background workers
+ *
+ * If streaming = parallel, we assign a new apply background worker (if
+ * available) as soon as the xact's first stream is received. The main apply
+ * worker will send changes to this new worker via shared memory. We keep this
+ * worker assigned till the transaction commit is received and also wait for
+ * the worker to finish at commit. This preserves commit ordering and avoids
+ * file I/O in most cases. We still need to spill to a file if there is no
+ * worker available. It is important to maintain commit order to avoid failures
+ * due to (a) transaction dependencies, say if we insert a row in the first
+ * transaction and update it in the second transaction then allowing to apply
+ * both in parallel can lead to failure in the update. (b) deadlocks, allowing
+ * transactions that update the same set of rows/tables in opposite order to be
+ * applied in parallel can lead to deadlocks.
+ *
+ * 2) Write to temporary files and apply when the final commit arrives
+ *
+ * If no worker is available to handle streamed transaction, the data is
+ * written to temporary files and then applied at once when the final commit
+ * arrives.
*
* Unlike the regular (non-streamed) case, handling streamed transactions has
* to handle aborts of both the toplevel transaction and subtransactions. This
@@ -219,20 +239,8 @@ typedef struct ApplyExecutionData
PartitionTupleRouting *proute; /* partition routing info */
} ApplyExecutionData;
-/* Struct for saving and restoring apply errcontext information */
-typedef struct ApplyErrorCallbackArg
-{
- LogicalRepMsgType command; /* 0 if invalid */
- LogicalRepRelMapEntry *rel;
-
- /* Remote node information */
- int remote_attnum; /* -1 if invalid */
- TransactionId remote_xid;
- XLogRecPtr finish_lsn;
- char *origin_name;
-} ApplyErrorCallbackArg;
-
-static ApplyErrorCallbackArg apply_error_callback_arg =
+/* errcontext tracker */
+ApplyErrorCallbackArg apply_error_callback_arg =
{
.command = 0,
.rel = NULL,
@@ -242,7 +250,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg =
.origin_name = NULL,
};
-static MemoryContext ApplyMessageContext = NULL;
+MemoryContext ApplyMessageContext = NULL;
MemoryContext ApplyContext = NULL;
/* per stream context for streaming transactions */
@@ -251,27 +259,39 @@ static MemoryContext LogicalStreamingContext = NULL;
WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
Subscription *MySubscription = NULL;
-static bool MySubscriptionValid = false;
+bool MySubscriptionValid = false;
bool in_remote_transaction = false;
static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
/* fields valid only when processing streamed transaction */
-static bool in_streamed_transaction = false;
+bool in_streamed_transaction = false;
+
+TransactionId stream_xid = InvalidTransactionId;
+static ApplyBgworkerState *stream_apply_worker = NULL;
-static TransactionId stream_xid = InvalidTransactionId;
+/* Check if we are applying the transaction in an apply background worker */
+#define apply_bgworker_active() (in_streamed_transaction && stream_apply_worker != NULL)
+
+/*
+ * The number of changes during one streaming block (only for apply background
+ * workers)
+ */
+static uint32 nchanges = 0;
/*
* We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
* the subscription if the remote transaction's finish LSN matches the subskiplsn.
* Once we start skipping changes, we don't stop it until we skip all changes of
* the transaction even if pg_subscription is updated and MySubscription->skiplsn
- * gets changed or reset during that. Also, in streaming transaction cases, we
- * don't skip receiving and spooling the changes since we decide whether or not
+ * gets changed or reset during that. Also, in streaming transaction cases (streaming = on),
+ * we don't skip receiving and spooling the changes since we decide whether or not
* to skip applying the changes when starting to apply changes. The subskiplsn is
* cleared after successfully skipping the transaction or applying non-empty
* transaction. The latter prevents the mistakenly specified subskiplsn from
- * being left.
+ * being left. Note that we cannot skip the streaming transactions when using
+ * apply background workers because we cannot get the finish LSN before
+ * applying the changes.
*/
static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr;
#define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn)))
@@ -324,9 +344,6 @@ static void maybe_reread_subscription(void);
static void DisableSubscriptionAndExit(void);
-/* prototype needed because of stream_commit */
-static void apply_dispatch(StringInfo s);
-
static void apply_handle_commit_internal(LogicalRepCommitData *commit_data);
static void apply_handle_insert_internal(ApplyExecutionData *edata,
ResultRelInfo *relinfo,
@@ -359,7 +376,6 @@ static void stop_skipping_changes(void);
static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
/* Functions for apply error callback */
-static void apply_error_callback(void *arg);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
@@ -426,40 +442,85 @@ end_replication_step(void)
}
/*
- * Handle streamed transactions.
+ * Handle streamed transactions for both the main apply worker and the apply
+ * background workers.
+ *
+ * In streaming case (receiving a block of streamed transaction), for
+ * SUBSTREAM_ON mode, simply redirect it to a file for the proper toplevel
+ * transaction, and for SUBSTREAM_PARALLEL mode, send the changes to apply
+ * background workers (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes
+ * will also be applied in main apply worker).
*
- * If in streaming mode (receiving a block of streamed transaction), we
- * simply redirect it to a file for the proper toplevel transaction.
+ * For non-streamed transactions, returns false;
+ * For streamed transactions, returns true if in main apply worker, false
+ * otherwise.
*
- * Returns true for streamed transactions, false otherwise (regular mode).
+ * Exception: When the main apply worker is applying streaming transactions in
+ * parallel mode (e.g. when addressing LOGICAL_REP_MSG_RELATION or
+ * LOGICAL_REP_MSG_TYPE changes), then return false.
*/
static bool
handle_streamed_transaction(LogicalRepMsgType action, StringInfo s)
{
- TransactionId xid;
+ TransactionId current_xid = InvalidTransactionId;
- /* not in streaming mode */
- if (!in_streamed_transaction)
+ /* Not in streaming mode */
+ if (!(in_streamed_transaction || am_apply_bgworker()))
return false;
- Assert(stream_fd != NULL);
Assert(TransactionIdIsValid(stream_xid));
/*
* We should have received XID of the subxact as the first part of the
* message, so extract it.
*/
- xid = pq_getmsgint(s, 4);
+ current_xid = pq_getmsgint(s, 4);
- if (!TransactionIdIsValid(xid))
+ if (!TransactionIdIsValid(current_xid))
ereport(ERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("invalid transaction ID in streamed replication transaction")));
- /* Add the new subxact to the array (unless already there). */
- subxact_info_add(xid);
+ if (am_apply_bgworker())
+ {
+ /* Define a savepoint for a subxact if needed. */
+ apply_bgworker_subxact_info_add(current_xid);
+
+ return false;
+ }
+
+ if (apply_bgworker_active())
+ {
+ /*
+ * This is the main apply worker, but there is an apply background
+ * worker, so apply the changes of this transaction in that background
+ * worker. Pass the data to the worker.
+ */
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
+ nchanges += 1;
+
+ /*
+ * XXX The publisher side doesn't always send relation/type update
+ * messages after the streaming transaction, so also update the
+ * relation/type in main apply worker here. See function
+ * cleanup_rel_sync_cache.
+ */
+ if (action == LOGICAL_REP_MSG_RELATION ||
+ action == LOGICAL_REP_MSG_TYPE)
+ return false;
+
+ return true;
+ }
+
+ /*
+ * This is the main apply worker, but there is no apply background worker,
+ * so write to temporary files and apply when the final commit arrives.
+ *
+ * Add the new subxact to the array (unless already there).
+ */
+ subxact_info_add(current_xid);
- /* write the change to the current file */
+ /* Write the change to the current file */
stream_write_change(action, s);
return true;
@@ -844,6 +905,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -898,7 +962,9 @@ apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data)
* BeginTransactionBlock is necessary to balance the EndTransactionBlock
* called within the PrepareTransactionBlock below.
*/
- BeginTransactionBlock();
+ if (!IsTransactionBlock())
+ BeginTransactionBlock();
+
CommitTransactionCommand(); /* Completes the preceding Begin command. */
/*
@@ -950,6 +1016,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1064,10 +1133,6 @@ apply_handle_rollback_prepared(StringInfo s)
/*
* Handle STREAM PREPARE.
- *
- * Logic is in two parts:
- * 1. Replay all the spooled operations
- * 2. Mark the transaction as prepared
*/
static void
apply_handle_stream_prepare(StringInfo s)
@@ -1088,24 +1153,78 @@ apply_handle_stream_prepare(StringInfo s)
logicalrep_read_stream_prepare(s, &prepare_data);
set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn);
- elog(DEBUG1, "received prepare for streamed transaction %u", prepare_data.xid);
+ if (am_apply_bgworker())
+ {
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
- /* Replay all the spooled operations. */
- apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+ CommitTransactionCommand();
- /* Mark the transaction as prepared. */
- apply_handle_prepare_internal(&prepare_data);
+ pgstat_report_stat(false);
- CommitTransactionCommand();
+ list_free(subxactlist);
+ subxactlist = NIL;
- pgstat_report_stat(false);
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(prepare_data.xid);
- store_flush_position(prepare_data.end_lsn);
+ elog(DEBUG1, "received prepare for streamed transaction %u",
+ prepare_data.xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM PREPARE message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+
+ pgstat_report_stat(false);
+ store_flush_position(prepare_data.end_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
+
+ CommitTransactionCommand();
+
+ pgstat_report_stat(false);
+
+ store_flush_position(prepare_data.end_lsn);
+
+ in_remote_transaction = false;
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ }
+ }
in_remote_transaction = false;
+ stream_apply_worker = NULL;
- /* unlink the files with serialized changes and subxact info. */
- stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1155,15 +1274,6 @@ apply_handle_stream_start(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("duplicate STREAM START message")));
- /*
- * Start a transaction on stream start, this transaction will be committed
- * on the stream stop unless it is a tablesync worker in which case it
- * will be committed after processing all the messages. We need the
- * transaction for handling the buffile, used for serializing the
- * streaming data and subxact info.
- */
- begin_replication_step();
-
/* notify handle methods we're processing a remote transaction */
in_streamed_transaction = true;
@@ -1177,36 +1287,93 @@ apply_handle_stream_start(StringInfo s)
set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr);
- /*
- * Initialize the worker's stream_fileset if we haven't yet. This will be
- * used for the entire duration of the worker so create it in a permanent
- * context. We create this on the very first streaming message from any
- * transaction and then use it for this and other streaming transactions.
- * Now, we could create a fileset at the start of the worker as well but
- * then we won't be sure that it will ever be used.
- */
- if (MyLogicalRepWorker->stream_fileset == NULL)
+ if (am_apply_bgworker())
{
- MemoryContext oldctx;
-
- oldctx = MemoryContextSwitchTo(ApplyContext);
+ /*
+ * Make sure the handle apply_dispatch methods are aware we're in a
+ * remote transaction.
+ */
+ in_remote_transaction = true;
- MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
- FileSetInit(MyLogicalRepWorker->stream_fileset);
+ /* Begin the transaction. */
+ AcceptInvalidationMessages();
+ maybe_reread_subscription();
- MemoryContextSwitchTo(oldctx);
+ StartTransactionCommand();
+ BeginTransactionBlock();
+ CommitTransactionCommand();
}
+ else
+ {
+ /*
+ * This is the main apply worker. Check if there is any free apply
+ * background worker we can use to process this transaction.
+ */
+ if (first_segment)
+ stream_apply_worker = apply_bgworker_start(stream_xid);
+ else
+ stream_apply_worker = apply_bgworker_find(stream_xid);
- /* open the spool file for this transaction */
- stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+ if (stream_apply_worker)
+ {
+ /*
+ * If we have found a free worker or if we are already applying this
+ * transaction in an apply background worker, then we pass the data to
+ * that worker.
+ */
+ if (first_segment)
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
- /* if this is not the first segment, open existing subxact file */
- if (!first_segment)
- subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+ nchanges = 0;
+ elog(DEBUG1, "starting streaming of xid %u", stream_xid);
+ }
+ else
+ {
+ /*
+ * Since no apply background worker is available for the first
+ * stream start, serialize all the changes of the transaction.
+ *
+ * Start a transaction on stream start, this transaction will be
+ * committed on the stream stop unless it is a tablesync worker in
+ * which case it will be committed after processing all the
+ * messages. We need the transaction for handling the buffile,
+ * used for serializing the streaming data and subxact info.
+ */
+ begin_replication_step();
- pgstat_report_activity(STATE_RUNNING, NULL);
+ /*
+ * Initialize the worker's stream_fileset if we haven't yet. This will
+ * be used for the entire duration of the worker so create it in a
+ * permanent context. We create this on the very first streaming
+ * message from any transaction and then use it for this and other
+ * streaming transactions. Now, we could create a fileset at the start
+ * of the worker as well but then we won't be sure that it will ever
+ * be used.
+ */
+ if (MyLogicalRepWorker->stream_fileset == NULL)
+ {
+ MemoryContext oldctx;
- end_replication_step();
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
+ FileSetInit(MyLogicalRepWorker->stream_fileset);
+
+ MemoryContextSwitchTo(oldctx);
+ }
+
+ /* Open the spool file for this transaction. */
+ stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+
+ /* If this is not the first segment, open existing subxact file. */
+ if (!first_segment)
+ subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+
+ end_replication_step();
+ }
+ }
+
+ pgstat_report_activity(STATE_RUNNING, NULL);
}
/*
@@ -1220,53 +1387,52 @@ apply_handle_stream_stop(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("STREAM STOP message without STREAM START")));
- /*
- * Close the file with serialized changes, and serialize information about
- * subxacts for the toplevel transaction.
- */
- subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
- stream_close_file();
+ if (apply_bgworker_active())
+ {
+ char action = LOGICAL_REP_MSG_STREAM_STOP;
- /* We must be in a valid transaction state */
- Assert(IsTransactionState());
+ apply_bgworker_send_data(stream_apply_worker, 1, &action);
- /* Commit the per-stream transaction */
- CommitTransactionCommand();
+ elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
+ }
+ else
+ {
+ /*
+ * Close the file with serialized changes, and serialize information
+ * about subxacts for the toplevel transaction.
+ */
+ subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
+ stream_close_file();
- in_streamed_transaction = false;
+ /* We must be in a valid transaction state */
+ Assert(IsTransactionState());
- /* Reset per-stream context */
- MemoryContextReset(LogicalStreamingContext);
+ /* Commit the per-stream transaction */
+ CommitTransactionCommand();
+
+ /* Reset per-stream context */
+ MemoryContextReset(LogicalStreamingContext);
+ }
+
+ in_streamed_transaction = false;
+ stream_apply_worker = NULL;
pgstat_report_activity(STATE_IDLE, NULL);
reset_apply_error_context_info();
}
/*
- * Handle STREAM abort message.
+ * Handle STREAM ABORT message when the transaction was spilled to disk.
*/
static void
-apply_handle_stream_abort(StringInfo s)
+serialize_stream_abort(TransactionId xid, TransactionId subxid)
{
- TransactionId xid;
- TransactionId subxid;
-
- if (in_streamed_transaction)
- ereport(ERROR,
- (errcode(ERRCODE_PROTOCOL_VIOLATION),
- errmsg_internal("STREAM ABORT message without STREAM STOP")));
-
- logicalrep_read_stream_abort(s, &xid, &subxid);
-
/*
* If the two XIDs are the same, it's in fact abort of toplevel xact, so
* just delete the files with serialized info.
*/
if (xid == subxid)
- {
- set_apply_error_context_xact(xid, InvalidXLogRecPtr);
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
- }
else
{
/*
@@ -1290,8 +1456,6 @@ apply_handle_stream_abort(StringInfo s)
bool found = false;
char path[MAXPGPATH];
- set_apply_error_context_xact(subxid, InvalidXLogRecPtr);
-
subidx = -1;
begin_replication_step();
subxact_info_read(MyLogicalRepWorker->subid, xid);
@@ -1316,7 +1480,6 @@ apply_handle_stream_abort(StringInfo s)
cleanup_subxact_info();
end_replication_step();
CommitTransactionCommand();
- reset_apply_error_context_info();
return;
}
@@ -1339,6 +1502,143 @@ apply_handle_stream_abort(StringInfo s)
end_replication_step();
CommitTransactionCommand();
}
+}
+
+/*
+ * Handle STREAM ABORT message.
+ */
+static void
+apply_handle_stream_abort(StringInfo s)
+{
+ TransactionId xid;
+ TransactionId subxid;
+ LogicalRepStreamAbortData abort_data;
+ bool read_abort_lsn = false;
+
+ if (in_streamed_transaction)
+ ereport(ERROR,
+ (errcode(ERRCODE_PROTOCOL_VIOLATION),
+ errmsg_internal("STREAM ABORT message without STREAM STOP")));
+
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelShared->server_version >=
+ LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM;
+
+ logicalrep_read_stream_abort(s, &abort_data, read_abort_lsn);
+
+ xid = abort_data.xid;
+ subxid = abort_data.subxid;
+
+ set_apply_error_context_xact(subxid, abort_data.abort_lsn);
+
+ if (am_apply_bgworker())
+ {
+ elog(DEBUG1, "[Apply BGW #%u] aborting current transaction xid=%u, subxid=%u",
+ MyParallelShared->n, GetCurrentTransactionIdIfAny(),
+ GetCurrentSubTransactionId());
+
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ if (read_abort_lsn)
+ {
+ replorigin_session_origin_lsn = abort_data.abort_lsn;
+ replorigin_session_origin_timestamp = abort_data.abort_time;
+ }
+
+ /*
+ * If the two XIDs are the same, it's in fact abort of toplevel xact,
+ * so just free the subxactlist.
+ */
+ if (subxid == xid)
+ {
+ AbortCurrentTransaction();
+
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
+
+ in_remote_transaction = false;
+ pgstat_report_activity(STATE_IDLE, NULL);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /*
+ * OK, so it's a subxact. Rollback to the savepoint.
+ *
+ * We also need to read the subxactlist, determine the offset
+ * tracked for the subxact, and truncate the list.
+ */
+ int i;
+ bool found = false;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", subxid);
+
+ elog(DEBUG1, "[Apply BGW #%u] rolling back to savepoint %s",
+ MyParallelShared->n, spname);
+
+ for (i = list_length(subxactlist) - 1; i >= 0; i--)
+ {
+ xid = (TransactionId) list_nth_int(subxactlist, i);
+ if (xid == subxid)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if (found)
+ {
+ RollbackToSavepoint(spname);
+ CommitTransactionCommand();
+ subxactlist = list_truncate(subxactlist, i + 1);
+ }
+
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ }
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM ABORT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ if (subxid == xid)
+ {
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+ }
+ }
+ else
+ {
+ /*
+ * We are in main apply worker and the transaction has been
+ * serialized to file.
+ */
+ serialize_stream_abort(xid, subxid);
+ }
+ }
reset_apply_error_context_info();
}
@@ -1468,8 +1768,8 @@ apply_spooled_messages(TransactionId xid, XLogRecPtr lsn)
static void
apply_handle_stream_commit(StringInfo s)
{
- TransactionId xid;
LogicalRepCommitData commit_data;
+ TransactionId xid;
if (in_streamed_transaction)
ereport(ERROR,
@@ -1479,14 +1779,81 @@ apply_handle_stream_commit(StringInfo s)
xid = logicalrep_read_stream_commit(s, &commit_data);
set_apply_error_context_xact(xid, commit_data.commit_lsn);
- elog(DEBUG1, "received commit for streamed transaction %u", xid);
+ if (am_apply_bgworker())
+ {
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ replorigin_session_origin_lsn = commit_data.end_lsn;
+ replorigin_session_origin_timestamp = commit_data.committime;
- apply_spooled_messages(xid, commit_data.commit_lsn);
+ CommitTransactionCommand();
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
- apply_handle_commit_internal(&commit_data);
+ in_remote_transaction = false;
+
+ pgstat_report_stat(false);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ elog(DEBUG1, "received commit for streamed transaction %u", xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM COMMIT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
- /* unlink the files with serialized changes and subxact info */
- stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ pgstat_report_stat(false);
+ store_flush_position(commit_data.end_lsn);
+ stop_skipping_changes();
+
+ apply_bgworker_free(wstate);
+
+ /*
+ * The transaction is either non-empty or skipped, so we clear the
+ * subskiplsn.
+ */
+ clear_subscription_skip_lsn(commit_data.commit_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(xid, commit_data.commit_lsn);
+
+ apply_handle_commit_internal(&commit_data);
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ }
+ }
+
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -2467,7 +2834,7 @@ apply_handle_truncate(StringInfo s)
/*
* Logical replication protocol message dispatcher.
*/
-static void
+void
apply_dispatch(StringInfo s)
{
LogicalRepMsgType action = pq_getmsgbyte(s);
@@ -2636,6 +3003,10 @@ store_flush_position(XLogRecPtr remote_lsn)
{
FlushPosition *flushpos;
+ /* Skip if not the main apply worker */
+ if (am_apply_bgworker())
+ return;
+
/* Need to do this in permanent context */
MemoryContextSwitchTo(ApplyContext);
@@ -2650,7 +3021,7 @@ store_flush_position(XLogRecPtr remote_lsn)
/* Update statistics of the worker. */
-static void
+void
UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply)
{
MyLogicalRepWorker->last_lsn = last_lsn;
@@ -2812,6 +3183,9 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
AcceptInvalidationMessages();
maybe_reread_subscription();
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any table synchronization changes. */
process_syncing_tables(last_received);
}
@@ -3113,7 +3487,7 @@ maybe_reread_subscription(void)
/*
* Callback from subscription syscache invalidation.
*/
-static void
+void
subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue)
{
MySubscriptionValid = false;
@@ -3709,7 +4083,7 @@ ApplyWorkerMain(Datum main_arg)
originid = replorigin_by_name(originname, true);
if (!OidIsValid(originid))
originid = replorigin_create(originname);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
origin_startpos = replorigin_session_get_progress(false);
CommitTransactionCommand();
@@ -3750,13 +4124,14 @@ ApplyWorkerMain(Datum main_arg)
server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
options.proto.logical.proto_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
LOGICALREP_PROTO_VERSION_NUM;
options.proto.logical.publication_names = MySubscription->publications;
options.proto.logical.binary = MySubscription->binary;
- options.proto.logical.streaming = MySubscription->stream;
+ options.proto.logical.streaming = (MySubscription->stream != SUBSTREAM_OFF);
options.proto.logical.twophase = false;
if (!am_tablesync_worker())
@@ -3914,7 +4289,8 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
XLogRecPtr myskiplsn = MySubscription->skiplsn;
bool started_tx = false;
- if (likely(XLogRecPtrIsInvalid(myskiplsn)))
+ if (likely(XLogRecPtrIsInvalid(myskiplsn)) ||
+ am_apply_bgworker())
return;
if (!IsTransactionState())
@@ -3986,7 +4362,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
}
/* Error callback to give more context info about the change being applied */
-static void
+void
apply_error_callback(void *arg)
{
ApplyErrorCallbackArg *errarg = &apply_error_callback_arg;
@@ -4014,23 +4390,47 @@ apply_error_callback(void *arg)
errarg->remote_xid,
LSN_FORMAT_ARGS(errarg->finish_lsn));
}
- else if (errarg->remote_attnum < 0)
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
else
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->rel->remoterel.attnames[errarg->remote_attnum],
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
+ {
+ if (errarg->remote_attnum < 0)
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ else
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ }
}
/* Set transaction information of apply error callback */
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index ba8a24d099..de29dc6da9 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -1818,6 +1818,9 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
XLogRecPtr abort_lsn)
{
ReorderBufferTXN *toptxn;
+ PGOutputData *data = (PGOutputData *) ctx->output_plugin_private;
+ bool write_abort_lsn = (data->protocol_version >=
+ LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM);
/*
* The abort should happen outside streaming block, even for streamed
@@ -1831,7 +1834,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
Assert(rbtxn_is_streamed(toptxn));
OutputPluginPrepareWrite(ctx, true);
- logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid);
+ logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn, abort_lsn,
+ write_abort_lsn);
OutputPluginWrite(ctx, true);
cleanup_rel_sync_cache(toptxn->xid, false);
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index da57a93034..2e146fe087 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -388,6 +388,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT:
event_name = "HashGrowBucketsReinsert";
break;
+ case WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE:
+ event_name = "LogicalApplyWorkerStateChange";
+ break;
case WAIT_EVENT_LOGICAL_SYNC_DATA:
event_name = "LogicalSyncData";
break;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d43..4284bcbcd1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3220,6 +3220,18 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_apply_bgworkers_per_subscription",
+ PGC_SIGHUP,
+ REPLICATION_SUBSCRIBERS,
+ gettext_noop("Maximum number of apply background workers per subscription."),
+ NULL,
+ },
+ &max_apply_bgworkers_per_subscription,
+ 2, 0, MAX_BACKENDS,
+ NULL, NULL, NULL
+ },
+
{
{"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Sets the amount of time to wait before forcing "
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..ad18710af4 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,7 @@
#max_logical_replication_workers = 4 # taken from max_worker_processes
# (change requires restart)
#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers
+#max_apply_bgworkers_per_subscription = 2 # taken from max_logical_replication_workers
#------------------------------------------------------------------------------
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e4fdb6b75b..9099fe5f74 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4456,7 +4456,7 @@ getSubscriptions(Archive *fout)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query, " s.substream,\n");
else
- appendPQExpBufferStr(query, " false AS substream,\n");
+ appendPQExpBufferStr(query, " 'f' AS substream,\n");
if (fout->remoteVersion >= 150000)
appendPQExpBufferStr(query,
@@ -4586,8 +4586,10 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo)
if (strcmp(subinfo->subbinary, "t") == 0)
appendPQExpBufferStr(query, ", binary = true");
- if (strcmp(subinfo->substream, "f") != 0)
+ if (strcmp(subinfo->substream, "t") == 0)
appendPQExpBufferStr(query, ", streaming = on");
+ else if (strcmp(subinfo->substream, "p") == 0)
+ appendPQExpBufferStr(query, ", streaming = parallel");
if (strcmp(subinfo->subtwophasestate, two_phase_disabled) != 0)
appendPQExpBufferStr(query, ", two_phase = on");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d1260f590c..d54540f5f5 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -68,7 +68,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subbinary; /* True if the subscription wants the
* publisher to send data in binary */
- bool substream; /* Stream in-progress transactions. */
+ char substream; /* Stream in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char subtwophasestate; /* Stream two-phase transactions */
@@ -109,7 +110,8 @@ typedef struct Subscription
bool enabled; /* Indicates if the subscription is enabled */
bool binary; /* Indicates if the subscription wants data in
* binary format */
- bool stream; /* Allow streaming in-progress transactions. */
+ char stream; /* Allow streaming in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char twophasestate; /* Allow streaming two-phase transactions */
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
@@ -120,6 +122,21 @@ typedef struct Subscription
List *publications; /* List of publication names to subscribe to */
} Subscription;
+/* Disallow streaming in-progress transactions */
+#define SUBSTREAM_OFF 'f'
+
+/*
+ * Streaming in-progress transactions are written to a temporary file and
+ * applied only after the transaction is committed on upstream.
+ */
+#define SUBSTREAM_ON 't'
+
+/*
+ * Streaming in-progress transactions are applied immediately via a background
+ * worker
+ */
+#define SUBSTREAM_PARALLEL 'p'
+
extern Subscription *GetSubscription(Oid subid, bool missing_ok);
extern void FreeSubscription(Subscription *sub);
extern void DisableSubscription(Oid subid);
diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h
index f1e2821e25..ac8ef94381 100644
--- a/src/include/replication/logicallauncher.h
+++ b/src/include/replication/logicallauncher.h
@@ -14,6 +14,7 @@
extern PGDLLIMPORT int max_logical_replication_workers;
extern PGDLLIMPORT int max_sync_workers_per_subscription;
+extern PGDLLIMPORT int max_apply_bgworkers_per_subscription;
extern void ApplyLauncherRegister(void);
extern void ApplyLauncherMain(Datum main_arg);
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index a771ab8ff3..eb0fd24fd8 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -32,12 +32,17 @@
*
* LOGICALREP_PROTO_TWOPHASE_VERSION_NUM is the minimum protocol version with
* support for two-phase commit decoding (at prepare time). Introduced in PG15.
+ *
+ * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version
+ * with support for streaming large transactions using apply background
+ * workers. Introduced in PG16.
*/
#define LOGICALREP_PROTO_MIN_VERSION_NUM 1
#define LOGICALREP_PROTO_VERSION_NUM 1
#define LOGICALREP_PROTO_STREAM_VERSION_NUM 2
#define LOGICALREP_PROTO_TWOPHASE_VERSION_NUM 3
-#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_TWOPHASE_VERSION_NUM
+#define LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM 4
+#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM
/*
* Logical message types
@@ -175,6 +180,17 @@ typedef struct LogicalRepRollbackPreparedTxnData
char gid[GIDSIZE];
} LogicalRepRollbackPreparedTxnData;
+/*
+ * Transaction protocol information for stream abort.
+ */
+typedef struct LogicalRepStreamAbortData
+{
+ TransactionId xid;
+ TransactionId subxid;
+ XLogRecPtr abort_lsn;
+ TimestampTz abort_time;
+} LogicalRepStreamAbortData;
+
extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn);
extern void logicalrep_read_begin(StringInfo in,
LogicalRepBeginData *begin_data);
@@ -246,9 +262,12 @@ extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn
extern TransactionId logicalrep_read_stream_commit(StringInfo out,
LogicalRepCommitData *commit_data);
extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid);
-extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid);
+ ReorderBufferTXN *txn,
+ XLogRecPtr abort_lsn,
+ bool write_abort_lsn);
+extern void logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn);
extern char *logicalrep_message_type(LogicalRepMsgType action);
#endif /* LOGICAL_PROTO_H */
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index cd1b6e8afc..6a1af7f13c 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -13,6 +13,7 @@
#define LOGICALWORKER_H
extern void ApplyWorkerMain(Datum main_arg);
+extern void ApplyBgworkerMain(Datum main_arg);
extern bool IsLogicalWorker(void);
diff --git a/src/include/replication/origin.h b/src/include/replication/origin.h
index 2d1b5e5c28..c7389b40a7 100644
--- a/src/include/replication/origin.h
+++ b/src/include/replication/origin.h
@@ -53,7 +53,7 @@ extern XLogRecPtr replorigin_get_progress(RepOriginId node, bool flush);
extern void replorigin_session_advance(XLogRecPtr remote_commit,
XLogRecPtr local_commit);
-extern void replorigin_session_setup(RepOriginId node);
+extern void replorigin_session_setup(RepOriginId node, bool must_acquire);
extern void replorigin_session_reset(void);
extern XLogRecPtr replorigin_session_get_progress(bool flush);
diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index d109d0baed..d2a80d79e5 100644
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -301,6 +301,7 @@ typedef struct ReorderBufferTXN
{
TimestampTz commit_time;
TimestampTz prepare_time;
+ TimestampTz abort_time;
} xact_time;
/*
@@ -647,9 +648,11 @@ extern void ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
extern void ReorderBufferAssignChild(ReorderBuffer *, TransactionId, TransactionId, XLogRecPtr commit_lsn);
extern void ReorderBufferCommitChild(ReorderBuffer *, TransactionId, TransactionId,
XLogRecPtr commit_lsn, XLogRecPtr end_lsn);
-extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferAbortOld(ReorderBuffer *, TransactionId xid);
-extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferInvalidate(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
extern void ReorderBufferSetBaseSnapshot(ReorderBuffer *, TransactionId, XLogRecPtr lsn, struct SnapshotData *snap);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..a3560d4904 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -17,8 +17,11 @@
#include "access/xlogdefs.h"
#include "catalog/pg_subscription.h"
#include "datatype/timestamp.h"
+#include "replication/logicalrelation.h"
#include "storage/fileset.h"
#include "storage/lock.h"
+#include "storage/shm_mq.h"
+#include "storage/shm_toc.h"
#include "storage/spin.h"
@@ -60,6 +63,9 @@ typedef struct LogicalRepWorker
*/
FileSet *stream_fileset;
+ /* Indicates if this slot is used for an apply background worker. */
+ bool subworker;
+
/* Stats. */
XLogRecPtr last_lsn;
TimestampTz last_send_time;
@@ -68,8 +74,68 @@ typedef struct LogicalRepWorker
TimestampTz reply_time;
} LogicalRepWorker;
+/* Struct for saving and restoring apply errcontext information */
+typedef struct ApplyErrorCallbackArg
+{
+ LogicalRepMsgType command; /* 0 if invalid */
+ LogicalRepRelMapEntry *rel;
+
+ /* Remote node information */
+ int remote_attnum; /* -1 if invalid */
+ TransactionId remote_xid;
+ XLogRecPtr finish_lsn;
+ char *origin_name;
+} ApplyErrorCallbackArg;
+
+/*
+ * Status of apply background worker.
+ */
+typedef enum ApplyBgworkerStatus
+{
+ APPLY_BGWORKER_BUSY = 0, /* assigned to a transaction */
+ APPLY_BGWORKER_FINISHED, /* transaction is completed */
+ APPLY_BGWORKER_EXIT /* exit */
+} ApplyBgworkerStatus;
+
+/*
+ * Struct for sharing information between apply main and apply background
+ * workers.
+ */
+typedef struct ApplyBgworkerShared
+{
+ slock_t mutex;
+
+ /* Status of apply background worker. */
+ ApplyBgworkerStatus status;
+
+ /* server version of publisher. */
+ uint32 server_version;
+
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
+
+/*
+ * Struct for maintaining an apply background worker.
+ */
+typedef struct ApplyBgworkerState
+{
+ shm_mq_handle *mq_handle;
+ dsm_segment *dsm_seg;
+ ApplyBgworkerShared volatile *shared;
+} ApplyBgworkerState;
+
/* Main memory context for apply worker. Permanent during worker lifetime. */
extern PGDLLIMPORT MemoryContext ApplyContext;
+extern PGDLLIMPORT MemoryContext ApplyMessageContext;
+
+extern PGDLLIMPORT ApplyErrorCallbackArg apply_error_callback_arg;
+
+extern PGDLLIMPORT bool MySubscriptionValid;
+
+extern PGDLLIMPORT volatile ApplyBgworkerShared *MyParallelShared;
+
+extern PGDLLIMPORT List *subxactlist;
/* libpqreceiver connection */
extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn;
@@ -79,18 +145,22 @@ extern PGDLLIMPORT Subscription *MySubscription;
extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker;
extern PGDLLIMPORT bool in_remote_transaction;
+extern PGDLLIMPORT bool in_streamed_transaction;
+extern PGDLLIMPORT TransactionId stream_xid;
extern void logicalrep_worker_attach(int slot);
extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
bool only_running);
extern List *logicalrep_workers_find(Oid subid, bool only_running);
-extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
- Oid userid, Oid relid);
+extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
+ Oid userid, Oid relid,
+ dsm_handle subworker_dsm);
extern void logicalrep_worker_stop(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
extern int logicalrep_sync_worker_count(Oid subid);
+extern int logicalrep_apply_bgworker_count(Oid subid);
extern void ReplicationOriginNameForTablesync(Oid suboid, Oid relid,
char *originname, int szorgname);
@@ -103,10 +173,38 @@ extern void process_syncing_tables(XLogRecPtr current_lsn);
extern void invalidate_syncing_table_states(Datum arg, int cacheid,
uint32 hashvalue);
+extern void UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time,
+ bool reply);
+
+extern void apply_dispatch(StringInfo s);
+
+/* Function for apply error callback */
+extern void apply_error_callback(void *arg);
+
+extern void subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue);
+
+/* Apply background worker setup and interactions */
+extern ApplyBgworkerState *apply_bgworker_start(TransactionId xid);
+extern ApplyBgworkerState *apply_bgworker_find(TransactionId xid);
+extern void apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status);
+extern void apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes,
+ const void *data);
+extern void apply_bgworker_free(ApplyBgworkerState *wstate);
+extern void apply_bgworker_check_status(void);
+extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
+extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+
static inline bool
am_tablesync_worker(void)
{
return OidIsValid(MyLogicalRepWorker->relid);
}
+static inline bool
+am_apply_bgworker(void)
+{
+ return MyLogicalRepWorker->subworker;
+}
+
#endif /* WORKER_INTERNAL_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index c3ade01120..e35f199fd4 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -105,6 +105,7 @@ typedef enum
WAIT_EVENT_HASH_GROW_BUCKETS_ALLOCATE,
WAIT_EVENT_HASH_GROW_BUCKETS_ELECT,
WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE,
WAIT_EVENT_LOGICAL_SYNC_DATA,
WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE,
WAIT_EVENT_MQ_INTERNAL,
diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out
index 5db7146e06..919266ae06 100644
--- a/src/test/regress/expected/subscription.out
+++ b/src/test/regress/expected/subscription.out
@@ -197,7 +197,7 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
DROP SUBSCRIPTION regress_testsub;
-- fail - streaming must be boolean
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
-ERROR: streaming requires a Boolean value
+ERROR: streaming requires a Boolean value or "parallel"
-- now it works
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 34a76ceb60..4137dc77b4 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -116,6 +116,10 @@ Append
AppendPath
AppendRelInfo
AppendState
+ApplyBgworkerEntry
+ApplyBgworkerShared
+ApplyBgworkerState
+ApplyBgworkerStatus
ApplyErrorCallbackArg
ApplyExecutionData
ApplySubXactData
@@ -1485,6 +1489,7 @@ LogicalRepRelId
LogicalRepRelMapEntry
LogicalRepRelation
LogicalRepRollbackPreparedTxnData
+LogicalRepStreamAbortData
LogicalRepTupleData
LogicalRepTyp
LogicalRepWorker
--
2.23.0.windows.1
[application/octet-stream] v18-0002-Test-streaming-parallel-option-in-tap-test.patch (69.1K, ../../OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-v18-0002-Test-streaming-parallel-option-in-tap-test.patch)
download | inline diff:
From 3d9d950d2e4c4b6ad644c5baffa293b48d21cac1 Mon Sep 17 00:00:00 2001
From: "shiy.fnst" <[email protected]>
Date: Fri, 13 May 2022 14:50:30 +0800
Subject: [PATCH v18 2/4] Test streaming parallel option in tap test
Change all TAP tests using the SUBSCRIPTION "streaming" parameter, so they
now test both 'on' and 'parallel' values.
---
src/test/subscription/t/015_stream.pl | 199 ++++---
src/test/subscription/t/016_stream_subxact.pl | 119 +++--
src/test/subscription/t/017_stream_ddl.pl | 188 ++++---
.../t/018_stream_subxact_abort.pl | 195 ++++---
.../t/019_stream_subxact_ddl_abort.pl | 110 +++-
.../subscription/t/022_twophase_cascade.pl | 363 +++++++------
.../subscription/t/023_twophase_stream.pl | 498 ++++++++++--------
7 files changed, 1035 insertions(+), 637 deletions(-)
diff --git a/src/test/subscription/t/015_stream.pl b/src/test/subscription/t/015_stream.pl
index 6561b189de..0bdd234935 100644
--- a/src/test/subscription/t/015_stream.pl
+++ b/src/test/subscription/t/015_stream.pl
@@ -8,6 +8,116 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Interleave a pair of transactions, each exceeding the 64kB limit.
+ my $in = '';
+ my $out = '';
+
+ my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
+
+ my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
+ on_error_stop => 0);
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ $in .= q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ };
+ $h->pump_nb;
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
+ DELETE FROM test_tab WHERE a > 5000;
+ COMMIT;
+ });
+
+ $in .= q{
+ COMMIT;
+ \q
+ };
+ $h->finish; # errors make the next test fail, so ignore them here
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'check extra columns contain local defaults');
+
+ # Test the streaming in binary mode
+ $node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain local defaults');
+
+ # Change the local values of the extra columns on the subscriber,
+ # update publisher, and check that subscriber retains the expected
+ # values. This is to ensure that non-streaming transactions behave
+ # properly after a streaming transaction.
+ $node_subscriber->safe_psql('postgres',
+ "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
+ );
+ $node_publisher->safe_psql('postgres',
+ "UPDATE test_tab SET b = md5(a::text)");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
+ );
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain locally changed data');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +147,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,82 +168,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Interleave a pair of transactions, each exceeding the 64kB limit.
-my $in = '';
-my $out = '';
-
-my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
-
-my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
- on_error_stop => 0);
-
-$in .= q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-};
-$h->pump_nb;
-
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
-DELETE FROM test_tab WHERE a > 5000;
-COMMIT;
-});
-
-$in .= q{
-COMMIT;
-\q
-};
-$h->finish; # errors make the next test fail, so ignore them here
-
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334), 'check extra columns contain local defaults');
-
-# Test the streaming in binary mode
-$node_subscriber->safe_psql('postgres',
- "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(6667|6667|6667), 'check extra columns contain local defaults');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+);
-# Change the local values of the extra columns on the subscriber,
-# update publisher, and check that subscriber retains the expected
-# values. This is to ensure that non-streaming transactions behave
-# properly after a streaming transaction.
$node_subscriber->safe_psql('postgres',
- "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
-);
-$node_publisher->safe_psql('postgres',
- "UPDATE test_tab SET b = md5(a::text)");
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel, binary = off)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
-);
-is($result, qq(6667|6667|6667),
- 'check extra columns contain locally changed data');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/016_stream_subxact.pl b/src/test/subscription/t/016_stream_subxact.pl
index f27f1694f2..45429dddba 100644
--- a/src/test/subscription/t/016_stream_subxact.pl
+++ b/src/test/subscription/t/016_stream_subxact.pl
@@ -8,6 +8,72 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(1667|1667|1667),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +103,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,41 +124,26 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Insert, update and delete enough rows to exceed 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(1667|1667|1667),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/017_stream_ddl.pl b/src/test/subscription/t/017_stream_ddl.pl
index 0bce63b716..52dfef4780 100644
--- a/src/test/subscription/t/017_stream_ddl.pl
+++ b/src/test/subscription/t/017_stream_ddl.pl
@@ -8,6 +8,111 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (3, md5(3::text));
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (4, md5(4::text), -4);
+ COMMIT;
+ });
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
+ is($result, qq(2002|1999|1002|1),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # A large (streamed) transaction with DDL and DML. One of the DDL is performed
+ # after DML to ensure that we invalidate the schema sent for test_tab so that
+ # the next transaction has to send the schema again.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
+ ALTER TABLE test_tab ADD COLUMN f INT;
+ COMMIT;
+ });
+
+ # A small transaction that won't get streamed. This is just to ensure that we
+ # send the schema again to reflect the last column added in the previous test.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab"
+ );
+ is($result, qq(5005|5002|4005|3004|5),
+ 'check data was copied to subscriber for both streaming and non-streaming transactions'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c, DROP COLUMN d, DROP COLUMN e, DROP COLUMN f;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +142,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,76 +163,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|0|0), 'check initial data was copied to subscriber');
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (3, md5(3::text));
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (4, md5(4::text), -4);
-COMMIT;
-});
-
-# large (streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
-COMMIT;
-});
-
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
-is($result, qq(2002|1999|1002|1),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# A large (streamed) transaction with DDL and DML. One of the DDL is performed
-# after DML to ensure that we invalidate the schema sent for test_tab so that
-# the next transaction has to send the schema again.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
-ALTER TABLE test_tab ADD COLUMN f INT;
-COMMIT;
-});
-
-# A small transaction that won't get streamed. This is just to ensure that we
-# send the schema again to reflect the last column added in the previous test.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
-COMMIT;
-});
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab");
-is($result, qq(5005|5002|4005|3004|5),
- 'check data was copied to subscriber for both streaming and non-streaming transactions'
-);
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/018_stream_subxact_abort.pl b/src/test/subscription/t/018_stream_subxact_abort.pl
index 7155442e76..68f0e4b0d1 100644
--- a/src/test/subscription/t/018_stream_subxact_abort.pl
+++ b/src/test/subscription/t/018_stream_subxact_abort.pl
@@ -8,6 +8,113 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
+ ROLLBACK TO s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
+ SAVEPOINT s5;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2000|0),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # large (streamed) transaction with subscriber receiving out of order
+ # subtransaction ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
+ RELEASE s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
+ ROLLBACK TO s1;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0),
+ 'check rollback to savepoint was reflected on subscriber');
+
+ # large (streamed) transaction with subscriber receiving rollback
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
+ ROLLBACK;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -36,6 +143,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -53,81 +164,25 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
-ROLLBACK TO s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
-SAVEPOINT s5;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2000|0),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# large (streamed) transaction with subscriber receiving out of order
-# subtransaction ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
-RELEASE s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
-ROLLBACK TO s1;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0),
- 'check rollback to savepoint was reflected on subscriber');
-
-# large (streamed) transaction with subscriber receiving rollback
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
-ROLLBACK;
-});
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
index dbd0fca4d1..b276063721 100644
--- a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
+++ b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
@@ -9,6 +9,69 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
+ ALTER TABLE test_tab DROP COLUMN c;
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(1000|500),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +100,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,35 +121,26 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
-ALTER TABLE test_tab DROP COLUMN c;
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(1000|500),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 7a797f37ba..0a4152d3be 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -11,6 +11,208 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming" parameter
+# so the same code can be run both for the streaming=on and streaming=parallel
+# cases.
+sub test_streaming
+{
+ my ($node_A, $node_B, $node_C, $appname_B, $appname_C, $streaming_mode) =
+ @_;
+
+ my $oldpid_B = $node_A->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';");
+ my $oldpid_C = $node_B->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';");
+
+ # Setup logical replication streaming mode
+
+ $node_B->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_B
+ SET (streaming = $streaming_mode);");
+ $node_C->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_C
+ SET (streaming = $streaming_mode)");
+
+ # Wait for subscribers to finish initialization
+
+ $node_A->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_B FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+ $node_B->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_C FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber(s) after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" optparameterion is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_B->reload;
+
+ $node_C->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_C->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ # Then 2PC PREPARE
+ $node_A->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_B->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_B->reload;
+
+ $node_C->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_C->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_C->reload;
+ }
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is prepared on subscriber(s)
+ my $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check that transaction was committed on subscriber(s)
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
+ );
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
+ );
+
+ # check the transaction state is ended on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber C');
+
+ ###############################
+ # Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
+ # 0. Cleanup from previous test leaving only 2 rows.
+ # 1. Insert one more row.
+ # 2. Record a SAVEPOINT.
+ # 3. Data is streamed using 2PC.
+ # 4. Do rollback to SAVEPOINT prior to the streamed inserts.
+ # 5. Then COMMIT PREPARED.
+ #
+ # Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (delete will be replicated)
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+
+ # 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
+ $node_A->safe_psql(
+ 'postgres', "
+ BEGIN;
+ INSERT INTO test_tab VALUES (9999, 'foobar');
+ SAVEPOINT sp_inner;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ ROLLBACK TO SAVEPOINT sp_inner;
+ PREPARE TRANSACTION 'outer';
+ ");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state prepared on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is ended on subscriber
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber C');
+
+ # check inserts are visible at subscriber(s).
+ # All the streamed data (prior to the SAVEPOINT) should be rolled back.
+ # (9999, 'foobar') should be committed.
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber B');
+ $result =
+ $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber B');
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber C');
+ $result =
+ $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber C');
+
+ # Cleanup the test data
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+}
+
###############################
# Setup a cascade of pub/sub nodes.
# node_A -> node_B -> node_C
@@ -260,160 +462,15 @@ is($result, qq(21), 'Rows committed are present on subscriber C');
# 2PC + STREAMING TESTS
# ---------------------
-my $oldpid_B = $node_A->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';");
-my $oldpid_C = $node_B->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';");
-
-# Setup logical replication (streaming = on)
-
-$node_B->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_B
- SET (streaming = on);");
-$node_C->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_C
- SET (streaming = on)");
-
-# Wait for subscribers to finish initialization
-
-$node_A->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_B FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-$node_B->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_C FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber(s) after the commit.
-###############################
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-# Then 2PC PREPARE
-$node_A->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
+################################
+# Test using streaming mode 'on'
+################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'on');
-# check the transaction state is prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check that transaction was committed on subscriber(s)
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
-);
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
-);
-
-# check the transaction state is ended on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber C');
-
-###############################
-# Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
-# 0. Cleanup from previous test leaving only 2 rows.
-# 1. Insert one more row.
-# 2. Record a SAVEPOINT.
-# 3. Data is streamed using 2PC.
-# 4. Do rollback to SAVEPOINT prior to the streamed inserts.
-# 5. Then COMMIT PREPARED.
-#
-# Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
-###############################
-
-# First, delete the data except for 2 rows (delete will be replicated)
-$node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
-$node_A->safe_psql(
- 'postgres', "
- BEGIN;
- INSERT INTO test_tab VALUES (9999, 'foobar');
- SAVEPOINT sp_inner;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- ROLLBACK TO SAVEPOINT sp_inner;
- PREPARE TRANSACTION 'outer';
- ");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state is ended on subscriber
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber C');
-
-# check inserts are visible at subscriber(s).
-# All the streamed data (prior to the SAVEPOINT) should be rolled back.
-# (9999, 'foobar') should be committed.
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber B');
-$result = $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber B');
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber C');
-$result = $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber C');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'parallel');
###############################
# check all the cleanup
diff --git a/src/test/subscription/t/023_twophase_stream.pl b/src/test/subscription/t/023_twophase_stream.pl
index d8475d25a4..b89414ab74 100644
--- a/src/test/subscription/t/023_twophase_stream.pl
+++ b/src/test/subscription/t/023_twophase_stream.pl
@@ -8,6 +8,266 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # check that 2PC gets replicated to subscriber
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ my $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ ###############################
+ # Test 2PC PREPARE / ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. Do rollback prepared.
+ #
+ # Expect data rolls back leaving only the original 2 rows.
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(2|2|2),
+ 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
+ # 1. insert, update and delete enough rows to exceed the 64kB limit.
+ # 2. Then server crashes before the 2PC transaction is committed.
+ # 3. After servers are restarted the pending transaction is committed.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ # Note: both publisher and subscriber do crash/restart.
+ ###############################
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_subscriber->stop('immediate');
+ $node_publisher->stop('immediate');
+
+ $node_publisher->start;
+ $node_subscriber->start;
+
+ # commit post the restart
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+ $node_publisher->wait_for_catchup($appname);
+
+ # check inserts are visible
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ ###############################
+ # Do INSERT after the PREPARE but before ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a ROLLBACK PREPARED.
+ #
+ # Expect the 2PC data rolls back leaving only 3 rows on the subscriber
+ # (the original 2 + inserted 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber,
+ # but the extra INSERT outside of the 2PC still was replicated
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3|3|3),
+ 'check the outside insert was copied to subscriber');
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Do INSERT after the PREPARE but before COMMIT PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a COMMIT PREPARED.
+ #
+ # Expect 2PC data + the extra row are on the subscriber
+ # (the 3334 + inserted 1 = 3335).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3335|3335|3335),
+ 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+ $node_publisher->wait_for_catchup($appname);
+}
+
###############################
# Setup
###############################
@@ -48,6 +308,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql(
'postgres', "
CREATE SUBSCRIPTION tap_sub
@@ -70,236 +334,30 @@ my $twophase_query =
$node_subscriber->poll_query_until('postgres', $twophase_query)
or die "Timed out while waiting for subscriber to enable twophase";
-###############################
# Check initial data was copied to subscriber
-###############################
my $result = $node_subscriber->safe_psql('postgres',
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber side after the commit.
-###############################
-
-# check that 2PC gets replicated to subscriber
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
-);
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
-
-###############################
-# Test 2PC PREPARE / ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. Do rollback prepared.
-#
-# Expect data rolls back leaving only the original 2 rows.
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(2|2|2),
- 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
-# 1. insert, update and delete enough rows to exceed the 64kB limit.
-# 2. Then server crashes before the 2PC transaction is committed.
-# 3. After servers are restarted the pending transaction is committed.
-#
-# Expect all data is replicated on subscriber side after the commit.
-# Note: both publisher and subscriber do crash/restart.
-###############################
-
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_subscriber->stop('immediate');
-$node_publisher->stop('immediate');
-
-$node_publisher->start;
-$node_subscriber->start;
-
-# commit post the restart
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-# check inserts are visible
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-###############################
-# Do INSERT after the PREPARE but before ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a ROLLBACK PREPARED.
-#
-# Expect the 2PC data rolls back leaving only 3 rows on the subscriber
-# (the original 2 + inserted 1).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber,
-# but the extra INSERT outside of the 2PC still was replicated
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3|3|3), 'check the outside insert was copied to subscriber');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Do INSERT after the PREPARE but before COMMIT PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a COMMIT PREPARED.
-#
-# Expect 2PC data + the extra row are on the subscriber
-# (the 3334 + inserted 1 = 3335).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3335|3335|3335),
- 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
-);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
###############################
# check all the cleanup
--
2.23.0.windows.1
[application/octet-stream] v18-0003-Add-some-checks-before-using-apply-background-wo.patch (37.8K, ../../OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-v18-0003-Add-some-checks-before-using-apply-background-wo.patch)
download | inline diff:
From 1751692945eb72ec332e79ccf28c88380ac8e2ae Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 11:23:52 +0800
Subject: [PATCH v18 3/4] Add some checks before using apply background worker
to apply changes.
streaming=parallel mode has two requirements:
1) The unique column in the relation on the subscriber-side should also be the
unique column on the publisher-side;
2) There cannot be any non-immutable functions used by the subscriber-side
replicated table. Look for functions in the following places:
* a. Trigger functions
* b. Column default value expressions and domain constraints
* c. Constraint expressions
* d. Foreign keys
---
doc/src/sgml/ref/create_subscription.sgml | 4 +
.../replication/logical/applybgworker.c | 44 ++
src/backend/replication/logical/proto.c | 88 +++-
src/backend/replication/logical/relation.c | 201 +++++++++
src/backend/replication/logical/tablesync.c | 1 +
src/backend/replication/logical/worker.c | 23 +-
src/backend/utils/cache/typcache.c | 17 +
src/include/replication/logicalproto.h | 1 +
src/include/replication/logicalrelation.h | 15 +
src/include/replication/worker_internal.h | 1 +
src/include/utils/typcache.h | 2 +
.../subscription/t/022_twophase_cascade.pl | 6 +
.../subscription/t/032_streaming_apply.pl | 380 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
14 files changed, 775 insertions(+), 9 deletions(-)
create mode 100644 src/test/subscription/t/032_streaming_apply.pl
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 71dd4aca81..bfd1895087 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -240,6 +240,10 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
transaction is committed. Note that if an error happens when
applying changes in a background worker, the finish LSN of the
remote transaction might not be reported in the server log.
+ <literal>parallel</literal> mode has two requirements: 1) the unique
+ column in the relation on the subscriber-side should also be the
+ unique column on the publisher-side; 2) there cannot be any
+ non-immutable functions used by the subscriber-side replicated table.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index aa222490a0..89c712f785 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -800,3 +800,47 @@ apply_bgworker_subxact_info_add(TransactionId current_xid)
MemoryContextSwitchTo(oldctx);
}
}
+
+/*
+ * Check if changes on this relation can be applied by an apply background
+ * worker.
+ *
+ * Although the commit order is maintained only allowing one process to commit
+ * at a time, the access order to the relation has changed. This could cause
+ * unexpected problems if the unique column on the replicated table is
+ * inconsistent with the publisher-side or contains non-immutable functions
+ * when applying transactions in the apply background worker.
+ */
+void
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
+{
+ /* Skip check if not an apply background worker. */
+ if (!am_apply_bgworker())
+ return;
+
+ /*
+ * Partition table checks are done later in function
+ * apply_handle_tuple_routing.
+ */
+ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ return;
+
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel_apply == PARALLEL_APPLY_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot replicate target relation \"%s.%s\" using "
+ "subscription parameter streaming=parallel",
+ rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change to use subscription parameter "
+ "streaming=on.")));
+}
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index 47bd811fb7..511bd9c052 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -23,7 +23,8 @@
/*
* Protocol message flags.
*/
-#define LOGICALREP_IS_REPLICA_IDENTITY 1
+#define ATTR_IS_REPLICA_IDENTITY (1 << 0)
+#define ATTR_IS_UNIQUE (1 << 1)
#define MESSAGE_TRANSACTIONAL (1<<0)
#define TRUNCATE_CASCADE (1<<0)
@@ -40,6 +41,68 @@ static void logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple);
static void logicalrep_write_namespace(StringInfo out, Oid nspid);
static const char *logicalrep_read_namespace(StringInfo in);
+static Bitmapset *RelationGetUniqueKeyBitmap(Relation rel);
+
+/*
+ * RelationGetUniqueKeyBitmap -- get a bitmap of unique attribute numbers
+ *
+ * This is similar to RelationGetIdentityKeyBitmap(), but returns a bitmap of
+ * index attribute numbers for all unique indexes.
+ */
+static Bitmapset *
+RelationGetUniqueKeyBitmap(Relation rel)
+{
+ List *indexoidlist = NIL;
+ ListCell *indexoidscan;
+ Bitmapset *attunique = NULL;
+
+ if (!rel->rd_rel->relhasindex)
+ return NULL;
+
+ indexoidlist = RelationGetIndexList(rel);
+
+ foreach(indexoidscan, indexoidlist)
+ {
+ Oid indexoid = lfirst_oid(indexoidscan);
+ Relation indexRel;
+ int i;
+
+ /* Look up the description for index */
+ indexRel = RelationIdGetRelation(indexoid);
+
+ if (!RelationIsValid(indexRel))
+ elog(ERROR, "could not open relation with OID %u", indexoid);
+
+ if (!indexRel->rd_index->indisunique)
+ {
+ RelationClose(indexRel);
+ continue;
+ }
+
+ /* Add referenced attributes to idindexattrs */
+ for (i = 0; i < indexRel->rd_index->indnatts; i++)
+ {
+ int attrnum = indexRel->rd_index->indkey.values[i];
+
+ /*
+ * We don't include non-key columns into idindexattrs
+ * bitmaps. See RelationGetIndexAttrBitmap.
+ */
+ if (attrnum != 0)
+ {
+ if (i < indexRel->rd_index->indnkeyatts &&
+ !bms_is_member(attrnum - FirstLowInvalidHeapAttributeNumber, attunique))
+ attunique = bms_add_member(attunique,
+ attrnum - FirstLowInvalidHeapAttributeNumber);
+ }
+ }
+ RelationClose(indexRel);
+ }
+ list_free(indexoidlist);
+
+ return attunique;
+}
+
/*
* Check if a column is covered by a column list.
*
@@ -933,7 +996,8 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
TupleDesc desc;
int i;
uint16 nliveatts = 0;
- Bitmapset *idattrs = NULL;
+ Bitmapset *idattrs = NULL,
+ *attunique = NULL;
bool replidentfull;
desc = RelationGetDescr(rel);
@@ -958,6 +1022,9 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (!replidentfull)
idattrs = RelationGetIdentityKeyBitmap(rel);
+ /* fetch bitmap of UNIQUE attributes */
+ attunique = RelationGetUniqueKeyBitmap(rel);
+
/* send the attributes */
for (i = 0; i < desc->natts; i++)
{
@@ -974,7 +1041,11 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (replidentfull ||
bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
idattrs))
- flags |= LOGICALREP_IS_REPLICA_IDENTITY;
+ flags |= ATTR_IS_REPLICA_IDENTITY;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ attunique))
+ flags |= ATTR_IS_UNIQUE;
pq_sendbyte(out, flags);
@@ -989,6 +1060,7 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
}
bms_free(idattrs);
+ bms_free(attunique);
}
/*
@@ -1001,7 +1073,8 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
int natts;
char **attnames;
Oid *atttyps;
- Bitmapset *attkeys = NULL;
+ Bitmapset *attkeys = NULL,
+ *attunique = NULL;
natts = pq_getmsgint(in, 2);
attnames = palloc(natts * sizeof(char *));
@@ -1014,9 +1087,13 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
/* Check for replica identity column */
flags = pq_getmsgbyte(in);
- if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
+ if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
+ /* Check for unique column */
+ if (flags & ATTR_IS_UNIQUE)
+ attunique = bms_add_member(attunique, i);
+
/* attribute name */
attnames[i] = pstrdup(pq_getmsgstring(in));
@@ -1030,6 +1107,7 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
rel->attnames = attnames;
rel->atttyps = atttyps;
rel->attkeys = attkeys;
+ rel->attunique = attunique;
rel->natts = natts;
}
diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c
index e989047681..37e410b8d0 100644
--- a/src/backend/replication/logical/relation.c
+++ b/src/backend/replication/logical/relation.c
@@ -19,12 +19,19 @@
#include "access/table.h"
#include "catalog/namespace.h"
+#include "catalog/pg_proc.h"
#include "catalog/pg_subscription_rel.h"
+#include "commands/trigger.h"
#include "executor/executor.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "replication/logicalrelation.h"
#include "replication/worker_internal.h"
+#include "rewrite/rewriteHandler.h"
#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
static MemoryContext LogicalRepRelMapContext = NULL;
@@ -91,6 +98,26 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
}
}
+/*
+ * Relcache invalidation callback to reset parallel flag.
+ */
+static void
+logicalrep_relmap_reset_parallel_cb(Datum arg, int cacheid, uint32 hashvalue)
+{
+ HASH_SEQ_STATUS hash_seq;
+ LogicalRepRelMapEntry *entry;
+
+ if (LogicalRepRelMap == NULL)
+ return;
+
+ hash_seq_init(&hash_seq, LogicalRepRelMap);
+ while ((entry = hash_seq_search(&hash_seq)) != NULL)
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNKNOWN;
+ entry->localrelvalid = false;
+ }
+}
+
/*
* Initialize the relation map cache.
*/
@@ -116,6 +143,9 @@ logicalrep_relmap_init(void)
/* Watch for invalidation events. */
CacheRegisterRelcacheCallback(logicalrep_relmap_invalidate_cb,
(Datum) 0);
+ CacheRegisterSyscacheCallback(PROCOID,
+ logicalrep_relmap_reset_parallel_cb,
+ (Datum) 0);
}
/*
@@ -142,6 +172,7 @@ logicalrep_relmap_free_entry(LogicalRepRelMapEntry *entry)
pfree(remoterel->atttyps);
}
bms_free(remoterel->attkeys);
+ bms_free(remoterel->attunique);
if (entry->attrmap)
free_attrmap(entry->attrmap);
@@ -190,6 +221,7 @@ logicalrep_relmap_update(LogicalRepRelation *remoterel)
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
MemoryContextSwitchTo(oldctx);
}
@@ -310,6 +342,168 @@ logicalrep_rel_mark_updatable(LogicalRepRelMapEntry *entry)
}
}
+/*
+ * Check if changes on one relation can be applied by an apply background
+ * worker and assign the 'parallel_apply' flag.
+ *
+ * There are two requirements for applying changes in an apply background
+ * worker: 1) The unique column in the relation on the subscriber-side should
+ * also be the unique column on the publisher-side; 2) There cannot be any
+ * non-immutable functions used by the subscriber-side.
+ *
+ * We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
+ * on one relation can not be applied by an apply background worker and leave
+ * it to apply_bgworker_relation_check() to throw the actual error if needed.
+ */
+static void
+logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
+{
+ Bitmapset *ukey;
+ int i;
+ TupleDesc tupdesc;
+ int attnum;
+ List *fkeys = NIL;
+
+ /* Fast path if 'parallel_apply' flag is already known. */
+ if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
+ return;
+
+ /* Initialize the flag. */
+ entry->parallel_apply = PARALLEL_APPLY_SAFE;
+
+ /*
+ * First, check if the unique column in the relation on the subscriber-side
+ * is also the unique column on the publisher-side.
+ */
+ ukey = RelationGetIndexAttrBitmap(entry->localrel,
+ INDEX_ATTR_BITMAP_KEY);
+
+ if (ukey)
+ {
+ i = -1;
+ while ((i = bms_next_member(ukey, i)) >= 0)
+ {
+ attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
+
+ if (entry->attrmap->attnums[attnum] < 0 ||
+ !bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+
+ bms_free(ukey);
+ }
+
+ /*
+ * Then, check if there is any non-immutable function used by the local
+ * table. Look for functions in the following places:
+ * a. trigger functions;
+ * b. Column default value expressions and domain constraints;
+ * c. Constraint expressions;
+ * d. Foreign keys.
+ */
+ /* Check the trigger functions. */
+ if (entry->localrel->trigdesc != NULL)
+ {
+ for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
+ {
+ Trigger *trig = entry->localrel->trigdesc->triggers + i;
+
+ if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
+ trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
+ continue;
+
+ if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the columns. */
+ tupdesc = RelationGetDescr(entry->localrel);
+ for (attnum = 0; attnum < tupdesc->natts; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
+
+ /* We don't need info for dropped or generated attributes */
+ if (att->attisdropped || att->attgenerated)
+ continue;
+
+ /*
+ * We don't need to check columns that only exist on the
+ * subscriber
+ */
+ if (entry->attrmap->attnums[attnum] < 0)
+ continue;
+
+ if (att->atthasdef)
+ {
+ Node *defaultexpr;
+
+ defaultexpr = build_column_default(entry->localrel, attnum + 1);
+ if (contain_mutable_functions(defaultexpr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+
+ /*
+ * If the column is of a DOMAIN type, determine whether
+ * that domain has any CHECK expressions that are not
+ * immutable.
+ */
+ if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
+ {
+ List *domain_constraints;
+ ListCell *lc;
+
+ domain_constraints = GetDomainConstraints(att->atttypid);
+
+ foreach(lc, domain_constraints)
+ {
+ DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
+
+ if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+ }
+
+ /* Check the constraints. */
+ if (tupdesc->constr)
+ {
+ ConstrCheck *check = tupdesc->constr->check;
+
+ /*
+ * Determine if there are any CHECK constraints which
+ * contains non-immutable function.
+ */
+ for (i = 0; i < tupdesc->constr->num_check; i++)
+ {
+ Expr *check_expr = stringToNode(check[i].ccbin);
+
+ if (contain_mutable_functions((Node *) check_expr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the foreign keys. */
+ fkeys = RelationGetFKeyList(entry->localrel);
+ if (fkeys)
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+}
+
/*
* Open the local relation associated with the remote one.
*
@@ -438,6 +632,9 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
*/
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel_apply(entry);
+
entry->localrelvalid = true;
}
@@ -653,6 +850,7 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
}
entry->localrel = partrel;
@@ -696,6 +894,9 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
/* Set if the table's replica identity is enough to apply update/delete. */
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel_apply(entry);
+
entry->localrelvalid = true;
/* state and statelsn are left set to 0. */
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8ffba7e2e5..3cdbf8b457 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -884,6 +884,7 @@ fetch_remote_table_info(char *nspname, char *relname,
lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
lrel->attkeys = NULL;
+ lrel->attunique = NULL;
/*
* Store the columns as a list of names. Ignore those that are not
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index b5aae0e19a..2216f6f3a7 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -1391,6 +1391,14 @@ apply_handle_stream_stop(StringInfo s)
{
char action = LOGICAL_REP_MSG_STREAM_STOP;
+ /*
+ * Unlike stream_commit, we don't need to wait here for stream_stop to
+ * finish. Allowing the other transaction to be applied before
+ * stream_stop is finished can lead to failures if the unique
+ * index/constraint is different between publisher and subscriber. But
+ * for such cases, we don't allow streamed transactions to be applied
+ * in parallel. See apply_bgworker_relation_check.
+ */
apply_bgworker_send_data(stream_apply_worker, 1, &action);
elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
@@ -2044,6 +2052,8 @@ apply_handle_insert(StringInfo s)
/* Set relation for error callback */
apply_error_callback_arg.rel = rel;
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2187,6 +2197,8 @@ apply_handle_update(StringInfo s)
/* Check if we can do the update. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2355,6 +2367,8 @@ apply_handle_delete(StringInfo s)
/* Check if we can do the delete. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2540,13 +2554,14 @@ apply_handle_tuple_routing(ApplyExecutionData *edata,
}
MemoryContextSwitchTo(oldctx);
+ part_entry = logicalrep_partition_open(relmapentry, partrel,
+ attrmap);
+
/* Check if we can do the update or delete on the leaf partition. */
if (operation == CMD_UPDATE || operation == CMD_DELETE)
- {
- part_entry = logicalrep_partition_open(relmapentry, partrel,
- attrmap);
check_relation_updatable(part_entry);
- }
+
+ apply_bgworker_relation_check(part_entry);
switch (operation)
{
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 808f9ebd0d..b248899d82 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -2540,6 +2540,23 @@ compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2)
return 0;
}
+/*
+ * GetDomainConstraints --- get DomainConstraintState list of specified domain type
+ */
+List *
+GetDomainConstraints(Oid type_id)
+{
+ TypeCacheEntry *typentry;
+ List *constraints = NIL;
+
+ typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
+
+ if(typentry->domainData != NULL)
+ constraints = typentry->domainData->constraints;
+
+ return constraints;
+}
+
/*
* Load (or re-load) the enumData member of the typcache entry.
*/
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index eb0fd24fd8..4395c11f75 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -113,6 +113,7 @@ typedef struct LogicalRepRelation
char replident; /* replica identity */
char relkind; /* remote relation kind */
Bitmapset *attkeys; /* Bitmap of key columns */
+ Bitmapset *attunique; /* Bitmap of unique columns */
} LogicalRepRelation;
/* Type mapping info */
diff --git a/src/include/replication/logicalrelation.h b/src/include/replication/logicalrelation.h
index 78cd7e77f5..8011e648d7 100644
--- a/src/include/replication/logicalrelation.h
+++ b/src/include/replication/logicalrelation.h
@@ -15,6 +15,19 @@
#include "access/attmap.h"
#include "replication/logicalproto.h"
+/*
+ * States to determine if changes on one relation can be applied using an
+ * apply background worker.
+ */
+typedef enum ParalleApplySafety
+{
+ PARALLEL_APPLY_UNKNOWN = 0, /* unknown */
+ PARALLEL_APPLY_SAFE, /* Can apply changes in an apply background
+ worker */
+ PARALLEL_APPLY_UNSAFE /* Can not apply changes in an apply background
+ worker */
+} ParalleApplySafety;
+
typedef struct LogicalRepRelMapEntry
{
LogicalRepRelation remoterel; /* key is remoterel.remoteid */
@@ -31,6 +44,8 @@ typedef struct LogicalRepRelMapEntry
Relation localrel; /* relcache entry (NULL when closed) */
AttrMap *attrmap; /* map of local attributes to remote ones */
bool updatable; /* Can apply updates/deletes? */
+ ParalleApplySafety parallel_apply; /* Can apply changes in an apply
+ background worker? */
/* Sync state. */
char state;
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index a3560d4904..1c0db05c8a 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -194,6 +194,7 @@ extern void apply_bgworker_free(ApplyBgworkerState *wstate);
extern void apply_bgworker_check_status(void);
extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+extern void apply_bgworker_relation_check(LogicalRepRelMapEntry *rel);
static inline bool
am_tablesync_worker(void)
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index 431ad7f1b3..ed7c2e7f48 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -199,6 +199,8 @@ extern uint64 assign_record_type_identifier(Oid type_id, int32 typmod);
extern int compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2);
+extern List *GetDomainConstraints(Oid type_id);
+
extern size_t SharedRecordTypmodRegistryEstimate(void);
extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 0a4152d3be..30a01f7305 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -39,6 +39,12 @@ sub test_streaming
ALTER SUBSCRIPTION tap_sub_C
SET (streaming = $streaming_mode)");
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_C->safe_psql(
+ 'postgres', "ALTER TABLE test_tab ALTER c DROP DEFAULT");
+ }
+
# Wait for subscribers to finish initialization
$node_A->poll_query_until(
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
new file mode 100644
index 0000000000..7f8bfa6745
--- /dev/null
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -0,0 +1,380 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test the restrictions of streaming mode "parallel" in logical replication
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $offset = 0;
+
+# Create publisher node
+my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->append_conf('postgresql.conf',
+ 'logical_decoding_work_mem = 64kB');
+$node_publisher->start;
+
+# Create subscriber node
+my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber');
+$node_subscriber->init;
+$node_subscriber->start;
+
+# Setup structure on publisher
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar)");
+
+# Setup structure on subscriber
+# We need to test normal table and partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar) PARTITION BY RANGE(a)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partition (LIKE test_tab_partitioned)");
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partitioned ATTACH PARTITION test_tab_partition DEFAULT"
+);
+
+# Setup logical replication
+my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub FOR TABLE test_tab");
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub_partitioned FOR TABLE test_tab_partitioned");
+
+my $appname = 'tap_sub';
+$node_subscriber->safe_psql(
+ 'postgres', "
+ CREATE SUBSCRIPTION tap_sub
+ CONNECTION '$publisher_connstr application_name=$appname'
+ PUBLICATION tap_pub, tap_pub_partitioned
+ WITH (streaming = parallel, copy_data = false)");
+
+$node_publisher->wait_for_catchup($appname);
+
+# It is not allowed that the unique index on the publisher and the subscriber
+# is different. Check the error reported by background worker in this case.
+# First we check the unique index on normal table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+my $result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Then we check the unique index on partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_partition_idx ON test_tab_partition (b)");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Triggers which execute non-immutable function are not allowed on the
+# subscriber side. Check the error reported by background worker in this case.
+# First we check the trigger function on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE FUNCTION trigger_func() RETURNS TRIGGER AS \$\$
+ BEGIN
+ RETURN NULL;
+ END
+\$\$ language plpgsql;
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# Then we check the trigger function on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab_partition
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab_partition ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# It is not allowed that column default value expression contains a
+# non-immutable function on the subscriber side. Check the error reported by
+# background worker in this case.
+# First we check the column default value expression on normal table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# Then we check the column default value expression on partition table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# It is not allowed that domain constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case.
+# Because the column type of the partition table must be the same as its parent
+# table, only test normal table here.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE DOMAIN test_domain AS int CHECK(VALUE > random());
+ALTER TABLE test_tab ALTER COLUMN a TYPE test_domain;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop domain constraint expression on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping domain constraint expression'
+);
+
+# It is not allowed that constraint expression contains a non-immutable function
+# on the subscriber side. Check the error reported by background worker in this
+# case.
+# First we check the constraint expression on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# Then we check the constraint expression on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# It is not allowed that foreign key on the subscriber side. Check the error
+# reported by background worker in this case.
+# First we check the foreign key on normal table.
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_f (a int primary key);
+ALTER TABLE test_tab ADD CONSTRAINT test_tabfk FOREIGN KEY(a) REFERENCES test_tab_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+# Then we check the foreign key on partition table.
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_partition_f (a int primary key);
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_patition_fk FOREIGN KEY(a) REFERENCES test_tab_partition_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+$node_subscriber->stop;
+$node_publisher->stop;
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 4137dc77b4..697e6a7ba3 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1891,6 +1891,7 @@ PageXLogRecPtr
PagetableEntry
Pairs
ParallelAppendState
+ParallelApplySafety
ParallelBitmapHeapState
ParallelBlockTableScanDesc
ParallelBlockTableScanWorker
--
2.23.0.windows.1
[application/octet-stream] v18-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch (29.2K, ../../OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-v18-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch)
download | inline diff:
From 20f9f5a8ecad44f8ab761a9078b4e00d52de5bc1 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 15 Jun 2022 11:02:08 +0800
Subject: [PATCH v18 4/4] Retry to apply streaming xact only in apply worker
When the subscription parameter is set streaming=parallel, the logic tries to
apply the streaming transaction using an apply background worker. If this
fails the background worker exits with an error.
In this case, retry applying the streaming transaction using the normal
streaming=on mode. This is done to avoid getting caught in a loop of the same
retry errors.
A new flag field "subretry" has been introduced to catalog "pg_subscription".
If the subscriber exits with an error, this flag will be set true, and
whenever the transaction is applied successfully, this flag is reset false.
Now, when deciding how to apply a streaming transaction, the logic can know if
this transaction has previously failed or not (by checking the "subretry"
field).
---
doc/src/sgml/catalogs.sgml | 9 +
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/catalog/pg_subscription.c | 1 +
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/subscriptioncmds.c | 1 +
.../replication/logical/applybgworker.c | 16 +-
src/backend/replication/logical/worker.c | 165 +++++++++++++-----
src/bin/pg_dump/pg_dump.c | 5 +-
src/include/catalog/pg_subscription.h | 4 +
.../subscription/t/032_streaming_apply.pl | 154 +++++++++-------
10 files changed, 254 insertions(+), 110 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 13c9e8eca1..816bcfe089 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7907,6 +7907,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>subretry</structfield> <type>bool</type>
+ </para>
+ <para>
+ True if the previous apply change failed, necessitating a retry
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>subconninfo</structfield> <type>text</type>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index bfd1895087..240a8331a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -244,6 +244,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
column in the relation on the subscriber-side should also be the
unique column on the publisher-side; 2) there cannot be any
non-immutable functions used by the subscriber-side replicated table.
+ When applying a streaming transaction, if either requirement is not
+ met, the background worker will exit with an error.
+ <literal>parallel</literal> mode is disregarded when retrying;
+ instead the transaction will be applied using <literal>on</literal>
+ mode.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index 8856ce3b50..9b7f09653d 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok)
sub->stream = subform->substream;
sub->twophasestate = subform->subtwophasestate;
sub->disableonerr = subform->subdisableonerr;
+ sub->retry = subform->subretry;
/* Get conninfo */
datum = SysCacheGetAttr(SUBSCRIPTIONOID,
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fedaed533b..10f4dd6785 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1298,8 +1298,8 @@ REVOKE ALL ON pg_replication_origin_status FROM public;
-- All columns of pg_subscription except subconninfo are publicly readable.
REVOKE ALL ON pg_subscription FROM public;
GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled,
- subbinary, substream, subtwophasestate, subdisableonerr, subslotname,
- subsynccommit, subpublications)
+ subbinary, substream, subtwophasestate, subdisableonerr,
+ subretry, subslotname, subsynccommit, subpublications)
ON pg_subscription TO public;
CREATE VIEW pg_stat_subscription_stats AS
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d4b2616ee6..612a83393a 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -662,6 +662,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
LOGICALREP_TWOPHASE_STATE_PENDING :
LOGICALREP_TWOPHASE_STATE_DISABLED);
values[Anum_pg_subscription_subdisableonerr - 1] = BoolGetDatum(opts.disableonerr);
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(false);
values[Anum_pg_subscription_subconninfo - 1] =
CStringGetTextDatum(conninfo);
if (opts.slot_name)
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index 89c712f785..5d7b55e5ab 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -106,6 +106,18 @@ apply_bgworker_can_start(TransactionId xid)
if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
return false;
+ /*
+ * Don't use apply background workers for retries, because it is possible
+ * that the last time we tried to apply a transaction using an apply
+ * background worker the checks failed (see function
+ * apply_bgworker_relation_check).
+ */
+ if (MySubscription->retry)
+ {
+ elog(DEBUG1, "apply background workers are not used for retries");
+ return false;
+ }
+
/*
* For streaming transactions that are being applied in apply background
* worker, we cannot decide whether to apply the change for a relation
@@ -840,7 +852,5 @@ apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
rel->remoterel.nspname, rel->remoterel.relname),
errdetail("The unique column on subscriber is not the unique "
"column on publisher or there is at least one "
- "non-immutable function."),
- errhint("Please change to use subscription parameter "
- "streaming=on.")));
+ "non-immutable function.")));
}
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 2216f6f3a7..9635a12fea 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -379,6 +379,8 @@ static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
+static void set_subscription_retry(bool retry);
+
/*
* Should this worker apply changes for given relation.
*
@@ -905,6 +907,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1016,6 +1021,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1069,6 +1077,9 @@ apply_handle_commit_prepared(StringInfo s)
store_flush_position(prepare_data.end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1124,6 +1135,9 @@ apply_handle_rollback_prepared(StringInfo s)
store_flush_position(rollback_data.rollback_end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(rollback_data.rollback_end_lsn);
@@ -1218,6 +1232,9 @@ apply_handle_stream_prepare(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
in_remote_transaction = false;
@@ -1646,6 +1663,9 @@ apply_handle_stream_abort(StringInfo s)
*/
serialize_stream_abort(xid, subxid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
reset_apply_error_context_info();
@@ -1858,6 +1878,9 @@ apply_handle_stream_commit(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
/* Check the status of apply background worker if any. */
@@ -3901,20 +3924,28 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed during table synchronization */
+ pgstat_report_subscription_error(MySubscription->oid, false);
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed during table synchronization. Abort
- * the current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, false);
- PG_RE_THROW();
- }
+ proc_exit(0);
}
PG_END_TRY();
@@ -3939,20 +3970,27 @@ start_apply(XLogRecPtr origin_startpos)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed while applying changes */
+ pgstat_report_subscription_error(MySubscription->oid,
+ !am_tablesync_worker());
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed while applying changes. Abort the
- * current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
-
- PG_RE_THROW();
- }
}
PG_END_TRY();
}
@@ -4198,28 +4236,11 @@ ApplyWorkerMain(Datum main_arg)
}
/*
- * After error recovery, disable the subscription in a new transaction
- * and exit cleanly.
+ * Disable the subscription in a new transaction.
*/
static void
DisableSubscriptionAndExit(void)
{
- /*
- * Emit the error message, and recover from the error state to an idle
- * state
- */
- HOLD_INTERRUPTS();
-
- EmitErrorReport();
- AbortOutOfAnyTransaction();
- FlushErrorState();
-
- RESUME_INTERRUPTS();
-
- /* Report the worker failed during either table synchronization or apply */
- pgstat_report_subscription_error(MyLogicalRepWorker->subid,
- !am_tablesync_worker());
-
/* Disable the subscription */
StartTransactionCommand();
DisableSubscription(MySubscription->oid);
@@ -4229,8 +4250,6 @@ DisableSubscriptionAndExit(void)
ereport(LOG,
errmsg("logical replication subscription \"%s\" has been disabled due to an error",
MySubscription->name));
-
- proc_exit(0);
}
/*
@@ -4465,3 +4484,63 @@ reset_apply_error_context_info(void)
apply_error_callback_arg.remote_attnum = -1;
set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
}
+
+/*
+ * Set subretry of pg_subscription catalog.
+ *
+ * If retry is true, subscriber is about to exit with an error. Otherwise, it
+ * means that the transaction was applied successfully.
+ */
+static void
+set_subscription_retry(bool retry)
+{
+ Relation rel;
+ HeapTuple tup;
+ bool started_tx = false;
+ bool nulls[Natts_pg_subscription];
+ bool replaces[Natts_pg_subscription];
+ Datum values[Natts_pg_subscription];
+
+ if (MySubscription->retry == retry ||
+ am_apply_bgworker())
+ return;
+
+ if (!IsTransactionState())
+ {
+ StartTransactionCommand();
+ started_tx = true;
+ }
+
+ /* Look up the subscription in the catalog */
+ rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+ tup = SearchSysCacheCopy1(SUBSCRIPTIONOID,
+ ObjectIdGetDatum(MySubscription->oid));
+
+ if (!HeapTupleIsValid(tup))
+ elog(ERROR, "subscription \"%s\" does not exist", MySubscription->name);
+
+ LockSharedObject(SubscriptionRelationId, MySubscription->oid, 0,
+ AccessShareLock);
+
+ /* Form a new tuple. */
+ memset(values, 0, sizeof(values));
+ memset(nulls, false, sizeof(nulls));
+ memset(replaces, false, sizeof(replaces));
+
+ /* Reset subretry */
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(retry);
+ replaces[Anum_pg_subscription_subretry - 1] = true;
+
+ tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+ replaces);
+
+ /* Update the catalog. */
+ CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+ /* Cleanup. */
+ heap_freetuple(tup);
+ table_close(rel, NoLock);
+
+ if (started_tx)
+ CommitTransactionCommand();
+}
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 9099fe5f74..f6e257d06d 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4478,8 +4478,9 @@ getSubscriptions(Archive *fout)
ntups = PQntuples(res);
/*
- * Get subscription fields. We don't include subskiplsn in the dump as
- * after restoring the dump this value may no longer be relevant.
+ * Get subscription fields. We don't include subskiplsn and subretry in
+ * the dump as after restoring the dump this value may no longer be
+ * relevant.
*/
i_tableoid = PQfnumber(res, "tableoid");
i_oid = PQfnumber(res, "oid");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index d54540f5f5..5f4e058ec1 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -76,6 +76,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subdisableonerr; /* True if a worker error should cause the
* subscription to be disabled */
+ bool subretry BKI_DEFAULT(f); /* True if the previous apply change failed. */
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* Connection string to the publisher */
text subconninfo BKI_FORCE_NOT_NULL;
@@ -116,6 +118,8 @@ typedef struct Subscription
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
* occurs */
+ bool retry; /* Indicates if the previous apply change
+ * failed. */
char *conninfo; /* Connection string to the publisher */
char *slotname; /* Name of the replication slot */
char *synccommit; /* Synchronous commit setting for worker */
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
index 7f8bfa6745..24c519a870 100644
--- a/src/test/subscription/t/032_streaming_apply.pl
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -57,8 +57,13 @@ $node_subscriber->safe_psql(
$node_publisher->wait_for_catchup($appname);
+# ============================================================================
# It is not allowed that the unique index on the publisher and the subscriber
-# is different. Check the error reported by background worker in this case.
+# is different. Check the error reported by background worker in this case. And
+# after retrying in apply worker, we check if the data is replicated
+# successfully.
+# ============================================================================
+
# First we check the unique index on normal table.
$node_subscriber->safe_psql('postgres',
"CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
@@ -71,14 +76,15 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
my $result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
# Then we check the unique index on partition table.
$node_subscriber->safe_psql('postgres',
@@ -95,17 +101,20 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
# Triggers which execute non-immutable function are not allowed on the
# subscriber side. Check the error reported by background worker in this case.
+# And after retrying in apply worker, we check if the data is replicated
+# successfully.
# First we check the trigger function on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -129,15 +138,16 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
# Then we check the trigger function on partition table.
$node_subscriber->safe_psql(
@@ -157,19 +167,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab_partition");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+# ============================================================================
# It is not allowed that column default value expression contains a
# non-immutable function on the subscriber side. Check the error reported by
-# background worker in this case.
+# background worker in this case. And after retrying in apply worker, we check
+# if the data is replicated successfully.
+# ============================================================================
+
# First we check the column default value expression on normal table.
$node_subscriber->safe_psql('postgres',
"ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
@@ -185,16 +200,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
# Then we check the column default value expression on partition table.
$node_subscriber->safe_psql('postgres',
@@ -211,20 +227,25 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+# ============================================================================
# It is not allowed that domain constraint expression contains a non-immutable
# function on the subscriber side. Check the error reported by background
-# worker in this case.
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# Because the column type of the partition table must be the same as its parent
# table, only test normal table here.
$node_subscriber->safe_psql(
@@ -242,21 +263,26 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop domain constraint expression on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(0),
- 'data replicated to subscriber after dropping domain constraint expression'
+ 'data replicated to subscribers after retrying because of domain'
);
-# It is not allowed that constraint expression contains a non-immutable function
-# on the subscriber side. Check the error reported by background worker in this
-# case.
+# Drop domain constraint expression on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+# ============================================================================
+# It is not allowed that constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# First we check the constraint expression on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -274,16 +300,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
# Then we check the constraint expression on partition table.
$node_subscriber->safe_psql(
@@ -300,19 +327,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(0),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+# ============================================================================
# It is not allowed that foreign key on the subscriber side. Check the error
-# reported by background worker in this case.
+# reported by background worker in this case. And after retrying in apply
+# worker, we check if the data is replicated successfully.
+# ============================================================================
+
# First we check the foreign key on normal table.
$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
$node_publisher->wait_for_catchup($appname);
@@ -333,16 +365,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
# Then we check the foreign key on partition table.
$node_publisher->wait_for_catchup($appname);
@@ -363,16 +396,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
$node_subscriber->stop;
$node_publisher->stop;
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-22 02:56 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 6 replies; 43+ messages in thread
From: [email protected] @ 2022-07-22 02:56 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> Attach the news patches.
Not able to apply patches cleanly because the change in HEAD (366283961a).
Therefore, I rebased the patch based on the changes in HEAD.
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] v19-0001-Perform-streaming-logical-transactions-by-backgr.patch (106.6K, ../../OS3PR01MB62752B29CEC2592E18236A9A9E909@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-v19-0001-Perform-streaming-logical-transactions-by-backgr.patch)
download | inline diff:
From 88fe01450d105be3b03c0a3e7bbefcda23df83b0 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Wed, 20 Apr 2022 16:45:07 +0800
Subject: [PATCH v19 1/4] Perform streaming logical transactions by background
workers
Currently, for large transactions, the publisher sends the data in multiple
streams (changes divided into chunks depending upon logical_decoding_work_mem),
and then on the subscriber-side, the apply worker writes the changes into
temporary files and once it receives the commit, it reads from the file and
applies the entire transaction. To improve the performance of such
transactions, we can instead allow them to be applied via background workers.
In this approach, we assign a new apply background worker (if available) as
soon as the xact's first stream is received and the main apply worker will send
changes to this new worker via shared memory. The apply background worker will
directly apply the change instead of writing it to temporary files. We keep
this worker assigned till the transaction commit is received and also wait for
the worker to finish at commit. This preserves commit ordering and avoids
writing to and reading from file in most cases. We still need to spill if there
is no worker available.
This patch also extends the SUBSCRIPTION 'streaming' parameter so that the user
can control whether to apply the streaming transaction in an apply background
worker or spill the change to disk. The user can set the streaming parameter to
'on/off', 'parallel'. The parameter value 'parallel' means the streaming will
be applied via an apply background worker, if available. The parameter value
'on' means the streaming transaction will be spilled to disk. The default value
is 'off' (same as current behaviour).
---
doc/src/sgml/catalogs.sgml | 10 +-
doc/src/sgml/config.sgml | 25 +
doc/src/sgml/logical-replication.sgml | 10 +
doc/src/sgml/protocol.sgml | 19 +
doc/src/sgml/ref/create_subscription.sgml | 24 +-
src/backend/access/transam/xact.c | 13 +
src/backend/commands/subscriptioncmds.c | 66 +-
src/backend/postmaster/bgworker.c | 3 +
src/backend/replication/logical/Makefile | 1 +
.../replication/logical/applybgworker.c | 802 ++++++++++++++++++
src/backend/replication/logical/decode.c | 10 +-
src/backend/replication/logical/launcher.c | 130 ++-
src/backend/replication/logical/origin.c | 26 +-
src/backend/replication/logical/proto.c | 41 +-
.../replication/logical/reorderbuffer.c | 10 +-
src/backend/replication/logical/tablesync.c | 10 +-
src/backend/replication/logical/worker.c | 692 +++++++++++----
src/backend/replication/pgoutput/pgoutput.c | 6 +-
src/backend/utils/activity/wait_event.c | 3 +
src/backend/utils/misc/guc.c | 12 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/bin/pg_dump/pg_dump.c | 6 +-
src/include/catalog/pg_subscription.h | 21 +-
src/include/replication/logicallauncher.h | 1 +
src/include/replication/logicalproto.h | 27 +-
src/include/replication/logicalworker.h | 1 +
src/include/replication/origin.h | 2 +-
src/include/replication/reorderbuffer.h | 7 +-
src/include/replication/worker_internal.h | 102 ++-
src/include/utils/wait_event.h | 1 +
src/test/regress/expected/subscription.out | 2 +-
src/tools/pgindent/typedefs.list | 5 +
32 files changed, 1869 insertions(+), 220 deletions(-)
create mode 100644 src/backend/replication/logical/applybgworker.c
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a186e35f00..099b3c9661 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7873,11 +7873,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry role="catalog_table_entry"><para role="column_definition">
- <structfield>substream</structfield> <type>bool</type>
+ <structfield>substream</structfield> <type>char</type>
</para>
<para>
- If true, the subscription will allow streaming of in-progress
- transactions
+ Controls how to handle the streaming of in-progress transactions:
+ <literal>f</literal> = disallow streaming of in-progress transactions,
+ <literal>t</literal> = spill the changes of in-progress transactions to
+ disk and apply at once after the transaction is committed on the
+ publisher,
+ <literal>p</literal> = apply changes directly using a background worker
</para></entry>
</row>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e2d728e0c4..a926b530ea 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4970,6 +4970,31 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
</listitem>
</varlistentry>
+ <varlistentry id="guc-max-apply-bgworkers-per-subscription" xreflabel="max_apply_bgworkers_per_subscription">
+ <term><varname>max_apply_bgworkers_per_subscription</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>max_apply_bgworkers_per_subscription</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Maximum number of apply background workers per subscription. This
+ parameter controls the amount of parallelism of the streaming of
+ in-progress transactions when subscription parameter
+ <literal>streaming = parallel</literal>.
+ </para>
+ <para>
+ The apply background workers are taken from the pool defined by
+ <varname>max_logical_replication_workers</varname>.
+ </para>
+ <para>
+ The default value is 2. This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command
+ line.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</sect2>
diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml
index bdf1e7b727..92997f9299 100644
--- a/doc/src/sgml/logical-replication.sgml
+++ b/doc/src/sgml/logical-replication.sgml
@@ -1153,6 +1153,16 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER
might not violate any constraint. This can easily make the subscriber
inconsistent.
</para>
+
+ <para>
+ When the streaming mode is <literal>parallel</literal>, the finish LSN of
+ failed transactions may not be logged. In that case, it may be necessary to
+ change the streaming mode to <literal>on</literal> and cause the same
+ conflicts again so the finish LSN of the failed transaction will be written
+ to the server log. For the usage of finish LSN, please refer to <link
+ linkend="sql-altersubscription"><command>ALTER SUBSCRIPTION ...
+ SKIP</command></link>.
+ </para>
</sect1>
<sect1 id="logical-replication-restrictions">
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..7e88ba9631 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -6809,6 +6809,25 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term>Int64 (XLogRecPtr)</term>
+ <listitem>
+ <para>
+ The LSN of the abort.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>Int64 (TimestampTz)</term>
+ <listitem>
+ <para>
+ Abort timestamp of the transaction. The value is in number
+ of microseconds since PostgreSQL epoch (2000-01-01).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term>Int32 (TransactionId)</term>
<listitem>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 7390c715bc..b08e4b5580 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -217,13 +217,29 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</varlistentry>
<varlistentry>
- <term><literal>streaming</literal> (<type>boolean</type>)</term>
+ <term><literal>streaming</literal> (<type>enum</type>)</term>
<listitem>
<para>
Specifies whether to enable streaming of in-progress transactions
- for this subscription. By default, all transactions
- are fully decoded on the publisher and only then sent to the
- subscriber as a whole.
+ for this subscription. The default value is <literal>off</literal>,
+ meaning all transactions are fully decoded on the publisher and only
+ then sent to the subscriber as a whole.
+ </para>
+
+ <para>
+ If set to <literal>on</literal>, the incoming changes are written to
+ temporary files and then applied only after the transaction is
+ committed on the publisher.
+ </para>
+
+ <para>
+ If set to <literal>parallel</literal>, incoming changes are directly
+ applied via one of the apply background workers, if available. If no
+ background worker is free to handle streaming transaction then the
+ changes are written to temporary files and applied after the
+ transaction is committed. Note that if an error happens when
+ applying changes in a background worker, the finish LSN of the
+ remote transaction might not be reported in the server log.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 116de1175b..3e61a57b50 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -1711,6 +1711,7 @@ RecordTransactionAbort(bool isSubXact)
int nchildren;
TransactionId *children;
TimestampTz xact_time;
+ bool replorigin;
/*
* If we haven't been assigned an XID, nobody will care whether we aborted
@@ -1741,6 +1742,13 @@ RecordTransactionAbort(bool isSubXact)
elog(PANIC, "cannot abort transaction %u, it was already committed",
xid);
+ /*
+ * Are we using the replication origins feature? Or, in other words,
+ * are we replaying remote actions?
+ */
+ replorigin = (replorigin_session_origin != InvalidRepOriginId &&
+ replorigin_session_origin != DoNotReplicateId);
+
/* Fetch the data we need for the abort record */
nrels = smgrGetPendingDeletes(false, &rels);
nchildren = xactGetCommittedChildren(&children);
@@ -1765,6 +1773,11 @@ RecordTransactionAbort(bool isSubXact)
MyXactFlags, InvalidTransactionId,
NULL);
+ if (replorigin)
+ /* Move LSNs forward for this replication origin */
+ replorigin_session_advance(replorigin_session_origin_lsn,
+ XactLastRecEnd);
+
/*
* Report the latest async abort LSN, so that the WAL writer knows to
* flush this abort. There's nothing to be gained by delaying this, since
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bd0cc0848d..9697128414 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -84,7 +84,7 @@ typedef struct SubOpts
bool copy_data;
bool refresh;
bool binary;
- bool streaming;
+ char streaming;
bool twophase;
bool disableonerr;
char *origin;
@@ -97,6 +97,62 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub,
static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
+/*
+ * Extract the streaming mode value from a DefElem. This is like
+ * defGetBoolean() but also accepts the special value of "parallel".
+ */
+static char
+defGetStreamingMode(DefElem *def)
+{
+ /*
+ * If no parameter value given, assume "true" is meant.
+ */
+ if (def->arg == NULL)
+ return SUBSTREAM_ON;
+
+ /*
+ * Allow 0, 1, "false", "true", "off", "on" or "parallel".
+ */
+ switch (nodeTag(def->arg))
+ {
+ case T_Integer:
+ switch (intVal(def->arg))
+ {
+ case 0:
+ return SUBSTREAM_OFF;
+ case 1:
+ return SUBSTREAM_ON;
+ default:
+ /* otherwise, error out below */
+ break;
+ }
+ break;
+ default:
+ {
+ char *sval = defGetString(def);
+
+ /*
+ * The set of strings accepted here should match up with the
+ * grammar's opt_boolean_or_string production.
+ */
+ if (pg_strcasecmp(sval, "false") == 0 ||
+ pg_strcasecmp(sval, "off") == 0)
+ return SUBSTREAM_OFF;
+ if (pg_strcasecmp(sval, "true") == 0 ||
+ pg_strcasecmp(sval, "on") == 0)
+ return SUBSTREAM_ON;
+ if (pg_strcasecmp(sval, "parallel") == 0)
+ return SUBSTREAM_PARALLEL;
+ }
+ break;
+ }
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("%s requires a Boolean value or \"parallel\"",
+ def->defname)));
+ return SUBSTREAM_OFF; /* keep compiler quiet */
+}
+
/*
* Common option parsing function for CREATE and ALTER SUBSCRIPTION commands.
*
@@ -134,7 +190,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
if (IsSet(supported_opts, SUBOPT_BINARY))
opts->binary = false;
if (IsSet(supported_opts, SUBOPT_STREAMING))
- opts->streaming = false;
+ opts->streaming = SUBSTREAM_OFF;
if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
opts->twophase = false;
if (IsSet(supported_opts, SUBOPT_DISABLE_ON_ERR))
@@ -237,7 +293,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options,
errorConflictingDefElem(defel, pstate);
opts->specified_opts |= SUBOPT_STREAMING;
- opts->streaming = defGetBoolean(defel);
+ opts->streaming = defGetStreamingMode(defel);
}
else if (strcmp(defel->defname, "two_phase") == 0)
{
@@ -627,7 +683,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner);
values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled);
values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary);
- values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming);
+ values[Anum_pg_subscription_substream - 1] = CharGetDatum(opts.streaming);
values[Anum_pg_subscription_subtwophasestate - 1] =
CharGetDatum(opts.twophase ?
LOGICALREP_TWOPHASE_STATE_PENDING :
@@ -1089,7 +1145,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
if (IsSet(opts.specified_opts, SUBOPT_STREAMING))
{
values[Anum_pg_subscription_substream - 1] =
- BoolGetDatum(opts.streaming);
+ CharGetDatum(opts.streaming);
replaces[Anum_pg_subscription_substream - 1] = true;
}
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 40601aefd9..40ccb8993c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -128,6 +128,9 @@ static const struct
},
{
"ApplyWorkerMain", ApplyWorkerMain
+ },
+ {
+ "ApplyBgworkerMain", ApplyBgworkerMain
}
};
diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile
index c4e2fdeb71..cbfb5d794e 100644
--- a/src/backend/replication/logical/Makefile
+++ b/src/backend/replication/logical/Makefile
@@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(srcdir) $(CPPFLAGS)
OBJS = \
+ applybgworker.o \
decode.o \
launcher.o \
logical.o \
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
new file mode 100644
index 0000000000..aa222490a0
--- /dev/null
+++ b/src/backend/replication/logical/applybgworker.c
@@ -0,0 +1,802 @@
+/*-------------------------------------------------------------------------
+ * applybgworker.c
+ * Support routines for applying xact by apply background worker
+ *
+ * Copyright (c) 2016-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/logical/applybgworker.c
+ *
+ * This file contains routines that are intended to support setting up, using,
+ * and tearing down a ApplyBgworkerState.
+ *
+ * Refer to the comments in file header of logical/worker.c to see more
+ * information about apply background worker.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "libpq/pqformat.h"
+#include "mb/pg_wchar.h"
+#include "pgstat.h"
+#include "postmaster/interrupt.h"
+#include "replication/logicallauncher.h"
+#include "replication/logicalworker.h"
+#include "replication/origin.h"
+#include "replication/walreceiver.h"
+#include "replication/worker_internal.h"
+#include "storage/ipc.h"
+#include "storage/procarray.h"
+#include "tcop/tcopprot.h"
+#include "utils/inval.h"
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+#include "utils/syscache.h"
+
+#define PG_LOGICAL_APPLY_SHM_MAGIC 0x79fb2447 // TODO Consider change
+
+/*
+ * DSM keys for apply background worker. Unlike other parallel execution code,
+ * since we don't need to worry about DSM keys conflicting with plan_node_id we
+ * can use small integers.
+ */
+#define APPLY_BGWORKER_KEY_SHARED 1
+#define APPLY_BGWORKER_KEY_MQ 2
+
+/* Queue size of DSM, 16 MB for now. */
+#define DSM_QUEUE_SIZE 160000000
+
+/*
+ * There are three fields in message: start_lsn, end_lsn and send_time. Because
+ * we have updated these statistics in apply worker, we could ignore these
+ * fields in apply background worker. (see function LogicalRepApplyLoop)
+ */
+#define IGNORE_SIZE_IN_MESSAGE (3 * sizeof(uint64))
+
+/*
+ * Entry for a hash table we use to map from xid to our apply background worker
+ * state.
+ */
+typedef struct ApplyBgworkerEntry
+{
+ TransactionId xid;
+ ApplyBgworkerState *wstate;
+} ApplyBgworkerEntry;
+
+/* Apply background workers hash table (initialized on first use) */
+static HTAB *ApplyWorkersHash = NULL;
+static List *ApplyWorkersFreeList = NIL;
+static List *ApplyWorkersList = NIL;
+
+/*
+ * Information shared between main apply worker and apply background worker.
+ */
+volatile ApplyBgworkerShared *MyParallelShared = NULL;
+
+List *subxactlist = NIL;
+
+static bool apply_bgworker_can_start(TransactionId xid);
+static ApplyBgworkerState *apply_bgworker_setup(void);
+static void apply_bgworker_setup_dsm(ApplyBgworkerState *wstate);
+
+/*
+ * Check if starting a new apply background worker is allowed.
+ */
+static bool
+apply_bgworker_can_start(TransactionId xid)
+{
+ if (!TransactionIdIsValid(xid))
+ return false;
+
+ /*
+ * Don't start a new background worker if not in streaming parallel mode.
+ */
+ if (MySubscription->stream != SUBSTREAM_PARALLEL)
+ return false;
+
+ /*
+ * Don't start a new background worker if user has set skiplsn as it's
+ * possible that user want to skip the streaming transaction. For
+ * streaming transaction, we need to spill the transaction to disk so that
+ * we can get the last LSN of the transaction to judge whether to skip
+ * before starting to apply the change.
+ */
+ if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
+ return false;
+
+ /*
+ * For streaming transactions that are being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time. So, we don't start new apply
+ * background worker in this case.
+ */
+ if (!AllTablesyncsReady())
+ return false;
+
+ return true;
+}
+
+/*
+ * Try to start an apply background worker and, if successful, cache it in
+ * ApplyWorkersHash keyed by the specified xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_start(TransactionId xid)
+{
+ bool found;
+ int server_version;
+ ApplyBgworkerState *wstate;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!apply_bgworker_can_start(xid))
+ return NULL;
+
+ /* First time through, initialize apply workers hashtable */
+ if (ApplyWorkersHash == NULL)
+ {
+ HASHCTL ctl;
+
+ MemSet(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(TransactionId);
+ ctl.entrysize = sizeof(ApplyBgworkerEntry);
+ ctl.hcxt = ApplyContext;
+
+ ApplyWorkersHash = hash_create("logical apply workers hash", 8, &ctl,
+ HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+ }
+
+ /*
+ * Now, we try to get an apply background worker. If there is at least one
+ * worker in the free list, then take one. Otherwise, we try to start a
+ * new apply background worker.
+ */
+ if (list_length(ApplyWorkersFreeList) > 0)
+ {
+ wstate = (ApplyBgworkerState *) llast(ApplyWorkersFreeList);
+ ApplyWorkersFreeList = list_delete_last(ApplyWorkersFreeList);
+ Assert(wstate->shared->status == APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ wstate = apply_bgworker_setup();
+
+ if (wstate == NULL)
+ return NULL;
+ }
+
+ /*
+ * Create entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_ENTER, &found);
+ if (found)
+ elog(ERROR, "hash table corrupted");
+
+ /* Fill up the hash entry */
+ wstate->shared->status = APPLY_BGWORKER_BUSY;
+
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ wstate->shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
+
+ wstate->shared->stream_xid = xid;
+ entry->wstate = wstate;
+ entry->xid = xid;
+
+ return wstate;
+}
+
+/*
+ * Try to look up worker inside ApplyWorkersHash for requested xid.
+ */
+ApplyBgworkerState *
+apply_bgworker_find(TransactionId xid)
+{
+ bool found;
+ ApplyBgworkerEntry *entry = NULL;
+
+ if (!TransactionIdIsValid(xid))
+ return NULL;
+
+ if (ApplyWorkersHash == NULL)
+ return NULL;
+
+ /*
+ * Find entry for requested transaction.
+ */
+ entry = hash_search(ApplyWorkersHash, &xid, HASH_FIND, &found);
+ if (found)
+ {
+ char status = entry->wstate->shared->status;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ entry->wstate->shared->n,
+ entry->wstate->shared->stream_xid)));
+
+ Assert(status == APPLY_BGWORKER_BUSY);
+
+ return entry->wstate;
+ }
+ else
+ return NULL;
+}
+
+/*
+ * Add the worker to the free list and remove the entry from the hash table.
+ */
+void
+apply_bgworker_free(ApplyBgworkerState *wstate)
+{
+ MemoryContext oldctx;
+ TransactionId xid = wstate->shared->stream_xid;
+
+ Assert(wstate->shared->status == APPLY_BGWORKER_FINISHED);
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ hash_search(ApplyWorkersHash, &xid, HASH_REMOVE, NULL);
+
+ elog(DEBUG1, "adding finished apply worker #%u for xid %u to the free list",
+ wstate->shared->n, wstate->shared->stream_xid);
+
+ ApplyWorkersFreeList = lappend(ApplyWorkersFreeList, wstate);
+
+ MemoryContextSwitchTo(oldctx);
+}
+
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *shared)
+{
+ shm_mq_result shmq_res;
+ PGPROC *registrant;
+ ErrorContextCallback errcallback;
+
+ registrant = BackendPidGetProc(MyBgworkerEntry->bgw_notify_pid);
+ SetLatch(®istrant->procLatch);
+
+ /*
+ * Push apply error context callback. Fields will be filled applying a
+ * change.
+ */
+ errcallback.callback = apply_error_callback;
+ errcallback.previous = error_context_stack;
+ error_context_stack = &errcallback;
+
+ for (;;)
+ {
+ void *data;
+ Size len;
+ int c;
+ StringInfoData s;
+ MemoryContext oldctx;
+
+ CHECK_FOR_INTERRUPTS();
+
+ /* Ensure we are reading the data into our memory context. */
+ oldctx = MemoryContextSwitchTo(ApplyMessageContext);
+
+ shmq_res = shm_mq_receive(mqh, &len, &data, false);
+
+ if (shmq_res != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("lost connection to the main apply worker")));
+
+ if (len == 0)
+ break;
+
+ s.cursor = 0;
+ s.maxlen = -1;
+ s.data = (char *) data;
+ s.len = len;
+
+ /*
+ * We use first byte of message for additional communication between
+ * main Logical replication worker and apply background workers, so if
+ * it differs from 'w', then process it first.
+ */
+ c = pq_getmsgbyte(&s);
+ switch (c)
+ {
+ /* End message of streaming chunk */
+ case LOGICAL_REP_MSG_STREAM_STOP:
+ elog(DEBUG1, "[Apply BGW #%u] ended processing streaming chunk,"
+ "waiting on shm_mq_receive", shared->n);
+
+ in_streamed_transaction = false;
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ continue;
+ case 'w':
+ break;
+ default:
+ elog(ERROR, "[Apply BGW #%u] unexpected message \"%c\"",
+ shared->n, c);
+ break;
+ }
+
+ /* Ignore statistics fields that have been updated. */
+ s.cursor += IGNORE_SIZE_IN_MESSAGE;
+
+ apply_dispatch(&s);
+
+ if (ConfigReloadPending)
+ {
+ ConfigReloadPending = false;
+ ProcessConfigFile(PGC_SIGHUP);
+ }
+
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
+ }
+
+ MemoryContextSwitchTo(TopMemoryContext);
+ MemoryContextReset(ApplyContext);
+
+ /* Pop the error context stack */
+ error_context_stack = errcallback.previous;
+
+ elog(DEBUG1, "[Apply BGW #%u] exiting", shared->n);
+
+ /* Signal main process that we are done. */
+ SetLatch(®istrant->procLatch);
+}
+
+/*
+ * Set the exit status so that the main apply worker can realize we have
+ * shutdown.
+ */
+static void
+apply_bgworker_shutdown(int code, Datum arg)
+{
+ SpinLockAcquire(&MyParallelShared->mutex);
+ MyParallelShared->status = APPLY_BGWORKER_EXIT;
+ SpinLockRelease(&MyParallelShared->mutex);
+
+ dsm_detach((dsm_segment *) DatumGetPointer(arg));
+}
+
+/*
+ * Apply Background Worker entry point
+ */
+void
+ApplyBgworkerMain(Datum main_arg)
+{
+ volatile ApplyBgworkerShared *shared;
+
+ dsm_handle handle;
+ dsm_segment *seg;
+ shm_toc *toc;
+ shm_mq *mq;
+ shm_mq_handle *mqh;
+ MemoryContext oldcontext;
+ RepOriginId originid;
+ int worker_slot = DatumGetInt32(main_arg);
+ char originname[NAMEDATALEN];
+
+ MemoryContextSwitchTo(TopMemoryContext);
+
+ /* Init the memory context for the apply background worker to work in. */
+ ApplyContext = AllocSetContextCreate(TopMemoryContext,
+ "ApplyContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Init the ApplyMessageContext which we clean up after each replication
+ * protocol message.
+ */
+ ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+ "ApplyMessageContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /* Setup signal handling */
+ pqsignal(SIGHUP, SignalHandlerForConfigReload);
+ pqsignal(SIGTERM, die);
+ BackgroundWorkerUnblockSignals();
+
+ /*
+ * Connect to the dynamic shared memory segment.
+ *
+ * The backend that registered this worker passed us the ID of a shared
+ * memory segment to which we must attach for further instructions. In
+ * order to attach to dynamic shared memory, we need a resource owner.
+ * Once we've mapped the segment in our address space, attach to the table
+ * of contents so we can locate the various data structures we'll need to
+ * find within the segment.
+ */
+ CurrentResourceOwner = ResourceOwnerCreate(NULL, "Logical apply worker");
+ memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsm_handle));
+ seg = dsm_attach(handle);
+ if (seg == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("unable to map dynamic shared memory segment")));
+ toc = shm_toc_attach(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg));
+ if (toc == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("bad magic number in dynamic shared memory segment")));
+
+ before_shmem_exit(apply_bgworker_shutdown, PointerGetDatum(seg));
+
+ /* Look up the shared information. */
+ shared = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_SHARED, false);
+ MyParallelShared = shared;
+
+ /*
+ * Attach to the message queue.
+ */
+ mq = shm_toc_lookup(toc, APPLY_BGWORKER_KEY_MQ, false);
+ shm_mq_set_receiver(mq, MyProc);
+ mqh = shm_mq_attach(mq, seg, NULL);
+
+ /* Run as replica session replication role. */
+ SetConfigOption("session_replication_role", "replica",
+ PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Now, we have initialized DSM. Attach to slot.
+ */
+ logicalrep_worker_attach(worker_slot);
+
+ /* Connect to our database. */
+ BackgroundWorkerInitializeConnectionByOid(MyLogicalRepWorker->dbid,
+ MyLogicalRepWorker->userid,
+ 0);
+
+ /*
+ * Set always-secure search path, so malicious users can't redirect user
+ * code (e.g. pg_index.indexprs).
+ */
+ SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
+
+ /*
+ * Set the client encoding to the database encoding, since that is what
+ * the leader will expect.
+ */
+ SetClientEncoding(GetDatabaseEncoding());
+
+ stream_xid = shared->stream_xid;
+
+ StartTransactionCommand();
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ MySubscription = GetSubscription(MyLogicalRepWorker->subid, true);
+ if (!MySubscription)
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply worker for subscription %u will not "
+ "start because the subscription was removed during startup",
+ MyLogicalRepWorker->subid)));
+ proc_exit(0);
+ }
+
+ MySubscriptionValid = true;
+ MemoryContextSwitchTo(oldcontext);
+
+ /* Setup synchronous commit according to the user's wishes */
+ SetConfigOption("synchronous_commit", MySubscription->synccommit,
+ PGC_BACKEND, PGC_S_OVERRIDE);
+
+ /* Keep us informed about subscription changes. */
+ CacheRegisterSyscacheCallback(SUBSCRIPTIONOID,
+ subscription_change_cb,
+ (Datum) 0);
+
+ CommitTransactionCommand();
+
+ /* Setup replication origin tracking. */
+ StartTransactionCommand();
+ snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
+ originid = replorigin_by_name(originname, true);
+ if (!OidIsValid(originid))
+ originid = replorigin_create(originname);
+
+ /*
+ * The apply background worker doesn't need to monopolize this replication
+ * origin which was already acquired by its leader process.
+ */
+ replorigin_session_setup(originid, false);
+ replorigin_session_origin = originid;
+ CommitTransactionCommand();
+
+ /*
+ * Allocate the origin name in long-lived context for error context
+ * message.
+ */
+ apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+ originname);
+
+ elog(DEBUG1, "[Apply BGW #%u] started", shared->n);
+
+ LogicalApplyBgwLoop(mqh, shared);
+
+ /*
+ * We're done. Explicitly detach the shared memory segment so that we
+ * don't get a resource leak warning at commit time. This will fire any
+ * on_dsm_detach callbacks we've registered, as well. Once that's done,
+ * we can go ahead and exit.
+ */
+ dsm_detach(seg);
+ proc_exit(0);
+}
+
+/*
+ * Set up a dynamic shared memory segment.
+ *
+ * We set up a control region that contains a ApplyBgworkerShared,
+ * plus one region per message queue. There are as many message queues as
+ * the number of workers.
+ */
+static void
+apply_bgworker_setup_dsm(ApplyBgworkerState *wstate)
+{
+ shm_toc_estimator e;
+ Size segsize;
+ dsm_segment *seg;
+ shm_toc *toc;
+ ApplyBgworkerShared *shared;
+ shm_mq *mq;
+ int64 queue_size = DSM_QUEUE_SIZE;
+ int server_version;
+
+ /*
+ * Estimate how much shared memory we need.
+ *
+ * Because the TOC machinery may choose to insert padding of oddly-sized
+ * requests, we must estimate each chunk separately.
+ *
+ * We need one key to register the location of the header, and we need
+ * another key to track the location of the message queue.
+ */
+ shm_toc_initialize_estimator(&e);
+ shm_toc_estimate_chunk(&e, sizeof(ApplyBgworkerShared));
+ shm_toc_estimate_chunk(&e, (Size) queue_size);
+
+ shm_toc_estimate_keys(&e, 1 + 1);
+ segsize = shm_toc_estimate(&e);
+
+ /* Create the shared memory segment and establish a table of contents. */
+ seg = dsm_create(shm_toc_estimate(&e), 0);
+ toc = shm_toc_create(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg),
+ segsize);
+
+ /* Set up the header region. */
+ shared = shm_toc_allocate(toc, sizeof(ApplyBgworkerShared));
+ SpinLockInit(&shared->mutex);
+ shared->status = APPLY_BGWORKER_BUSY;
+
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
+
+ shared->stream_xid = stream_xid;
+ shared->n = list_length(ApplyWorkersList) + 1;
+
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_SHARED, shared);
+
+ /* Set up message queue for the worker. */
+ mq = shm_mq_create(shm_toc_allocate(toc, (Size) queue_size),
+ (Size) queue_size);
+ shm_toc_insert(toc, APPLY_BGWORKER_KEY_MQ, mq);
+ shm_mq_set_sender(mq, MyProc);
+
+ /* Attach the queue. */
+ wstate->mq_handle = shm_mq_attach(mq, seg, NULL);
+
+ /* Return results to caller. */
+ wstate->dsm_seg = seg;
+ wstate->shared = shared;
+}
+
+/*
+ * Start apply background worker process and allocate shared memory for it.
+ */
+static ApplyBgworkerState *
+apply_bgworker_setup(void)
+{
+ MemoryContext oldcontext;
+ bool launched;
+ ApplyBgworkerState *wstate;
+ int napplyworkers;
+
+ elog(DEBUG1, "setting up apply worker #%u", list_length(ApplyWorkersList) + 1);
+
+ /* Check if there are free worker slot(s) */
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+ napplyworkers = logicalrep_apply_bgworker_count(MyLogicalRepWorker->subid);
+ LWLockRelease(LogicalRepWorkerLock);
+ if (napplyworkers >= max_apply_bgworkers_per_subscription)
+ return NULL;
+
+ oldcontext = MemoryContextSwitchTo(ApplyContext);
+
+ wstate = (ApplyBgworkerState *) palloc0(sizeof(ApplyBgworkerState));
+
+ /* Setup shared memory */
+ apply_bgworker_setup_dsm(wstate);
+
+ launched = logicalrep_worker_launch(MyLogicalRepWorker->dbid,
+ MySubscription->oid,
+ MySubscription->name,
+ MyLogicalRepWorker->userid,
+ InvalidOid,
+ dsm_segment_handle(wstate->dsm_seg));
+
+ if (launched)
+ ApplyWorkersList = lappend(ApplyWorkersList, wstate);
+ else
+ {
+ dsm_detach(wstate->dsm_seg);
+ wstate->dsm_seg = NULL;
+
+ pfree(wstate);
+ wstate = NULL;
+ }
+
+ MemoryContextSwitchTo(oldcontext);
+
+ return wstate;
+}
+
+/*
+ * Send the data to the specified apply background worker via shared-memory queue.
+ */
+void
+apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes, const void *data)
+{
+ shm_mq_result result;
+
+ result = shm_mq_send(wstate->mq_handle, nbytes, data, false, true);
+
+ if (result != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("could not send tuples to shared-memory queue")));
+}
+
+/*
+ * Wait until the status of apply background worker reaches the
+ * 'wait_for_status'
+ */
+void
+apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status)
+{
+ for (;;)
+ {
+ char status;
+
+ SpinLockAcquire(&wstate->shared->mutex);
+ status = wstate->shared->status;
+ SpinLockRelease(&wstate->shared->mutex);
+
+ /* Done if already in correct status. */
+ if (status == wait_for_status)
+ break;
+
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ wstate->shared->n, wstate->shared->stream_xid)));
+
+ /* Wait to be signalled. */
+ WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 0,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE);
+
+ /* Reset the latch so we don't spin. */
+ ResetLatch(MyLatch);
+
+ /* An interrupt may have occurred while we were waiting. */
+ CHECK_FOR_INTERRUPTS();
+ }
+}
+
+/*
+ * Check the status of workers and report an error if any apply background
+ * worker has exited unexpectedly.
+ */
+void
+apply_bgworker_check_status(void)
+{
+ ListCell *lc;
+
+ if (am_apply_bgworker() || MySubscription->stream != SUBSTREAM_PARALLEL)
+ return;
+
+ foreach(lc, ApplyWorkersList)
+ {
+ ApplyBgworkerState *wstate = (ApplyBgworkerState *) lfirst(lc);
+
+ /*
+ * We don't lock here as in the worst case we will just detect the
+ * failure of worker a bit later.
+ */
+ if (wstate->shared->status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u exited unexpectedly",
+ wstate->shared->n)));
+ }
+
+ /*
+ * Exit if any relation is not in the READY state and if any worker is
+ * handling the streaming transaction at the same time. Because for
+ * streaming transactions that is being applied in apply background
+ * worker, we cannot decide whether to apply the change for a relation
+ * that is not in the READY state (see should_apply_changes_for_rel) as we
+ * won't know remote_final_lsn by that time.
+ */
+ if (list_length(ApplyWorkersFreeList) != list_length(ApplyWorkersList) &&
+ !AllTablesyncsReady())
+ {
+ ereport(LOG,
+ (errmsg("logical replication apply workers for subscription \"%s\" will restart",
+ MySubscription->name),
+ errdetail("Cannot handle streamed replication transaction by apply "
+ "background workers until all tables are synchronized")));
+
+ proc_exit(0);
+ }
+}
+
+/* Set the apply background worker status */
+void
+apply_bgworker_set_status(ApplyBgworkerStatus status)
+{
+ if (!am_apply_bgworker())
+ return;
+
+ elog(DEBUG1, "[Apply BGW #%u] set status to %d", MyParallelShared->n, status);
+
+ SpinLockAcquire(&MyParallelShared->mutex);
+ MyParallelShared->status = status;
+ SpinLockRelease(&MyParallelShared->mutex);
+}
+
+/*
+ * Define a savepoint for a subxact in apply background worker if needed.
+ *
+ * Inside apply background worker we can figure out that new subtransaction was
+ * started if new change arrived with different xid. In that case we can define
+ * named savepoint, so that we were able to commit/rollback it separately
+ * later.
+ * Special case is if the first change comes from subtransaction, then
+ * we check that current_xid differs from stream_xid.
+ */
+void
+apply_bgworker_subxact_info_add(TransactionId current_xid)
+{
+ if (current_xid != stream_xid &&
+ !list_member_int(subxactlist, (int) current_xid))
+ {
+ MemoryContext oldctx;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", current_xid);
+
+ elog(DEBUG1, "[Apply BGW #%u] defining savepoint %s",
+ MyParallelShared->n, spname);
+
+ DefineSavepoint(spname);
+ CommitTransactionCommand();
+
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+ subxactlist = lappend_int(subxactlist, (int) current_xid);
+ MemoryContextSwitchTo(oldctx);
+ }
+}
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c5c6a2ba68..d4d5093a0b 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -651,9 +651,10 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
{
for (i = 0; i < parsed->nsubxacts; i++)
{
- ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr);
+ ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr,
+ commit_time);
}
- ReorderBufferForget(ctx->reorder, xid, buf->origptr);
+ ReorderBufferForget(ctx->reorder, xid, buf->origptr, commit_time);
return;
}
@@ -821,10 +822,11 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf,
for (i = 0; i < parsed->nsubxacts; i++)
{
ReorderBufferAbort(ctx->reorder, parsed->subxacts[i],
- buf->record->EndRecPtr);
+ buf->record->EndRecPtr, abort_time);
}
- ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr);
+ ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr,
+ abort_time);
}
/* update the decoding stats */
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 3bbd522724..d92bfaf6d6 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -54,6 +54,7 @@
int max_logical_replication_workers = 4;
int max_sync_workers_per_subscription = 2;
+int max_apply_bgworkers_per_subscription = 2;
LogicalRepWorker *MyLogicalRepWorker = NULL;
@@ -73,6 +74,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg);
static void logicalrep_worker_onexit(int code, Datum arg);
static void logicalrep_worker_detach(void);
static void logicalrep_worker_cleanup(LogicalRepWorker *worker);
+static void logicalrep_worker_stop_internal(LogicalRepWorker *worker);
static bool on_commit_launcher_wakeup = false;
@@ -151,8 +153,10 @@ get_subscription_list(void)
*
* This is only needed for cleaning up the shared memory in case the worker
* fails to attach.
+ *
+ * Return false if the attach fails. Otherwise return true.
*/
-static void
+static bool
WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
uint16 generation,
BackgroundWorkerHandle *handle)
@@ -168,11 +172,11 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
- /* Worker either died or has started; no need to do anything. */
+ /* Worker either died or has started. Return false if died. */
if (!worker->in_use || worker->proc)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return worker->in_use;
}
LWLockRelease(LogicalRepWorkerLock);
@@ -187,7 +191,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
if (generation == worker->generation)
logicalrep_worker_cleanup(worker);
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -223,6 +227,13 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running)
{
LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+ /*
+ * We are only interested in the main apply worker or table sync worker
+ * here.
+ */
+ if (w->subworker)
+ continue;
+
if (w->in_use && w->subid == subid && w->relid == relid &&
(!only_running || w->proc))
{
@@ -259,11 +270,11 @@ logicalrep_workers_find(Oid subid, bool only_running)
}
/*
- * Start new apply background worker, if possible.
+ * Start new background worker, if possible.
*/
-void
+bool
logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
- Oid relid)
+ Oid relid, dsm_handle subworker_dsm)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
@@ -273,6 +284,10 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
LogicalRepWorker *worker = NULL;
int nsyncworkers;
TimestampTz now;
+ bool is_subworker = (subworker_dsm != DSM_HANDLE_INVALID);
+
+ /* Sanity check : we don't support table sync in subworker. */
+ Assert(!(is_subworker && OidIsValid(relid)));
ereport(DEBUG1,
(errmsg_internal("starting logical replication worker for subscription \"%s\"",
@@ -350,7 +365,7 @@ retry:
if (OidIsValid(relid) && nsyncworkers >= max_sync_workers_per_subscription)
{
LWLockRelease(LogicalRepWorkerLock);
- return;
+ return false;
}
/*
@@ -364,7 +379,7 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of logical replication worker slots"),
errhint("You might need to increase max_logical_replication_workers.")));
- return;
+ return false;
}
/* Prepare the worker slot. */
@@ -379,6 +394,7 @@ retry:
worker->relstate = SUBREL_STATE_UNKNOWN;
worker->relstate_lsn = InvalidXLogRecPtr;
worker->stream_fileset = NULL;
+ worker->subworker = is_subworker;
worker->last_lsn = InvalidXLogRecPtr;
TIMESTAMP_NOBEGIN(worker->last_send_time);
TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -396,19 +412,31 @@ retry:
BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
- snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
+ if (is_subworker)
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyBgworkerMain");
+ else
+ snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+
if (OidIsValid(relid))
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u sync %u", subid, relid);
+ else if (is_subworker)
+ snprintf(bgw.bgw_name, BGW_MAXLEN,
+ "logical replication apply background worker for subscription %u", subid);
else
snprintf(bgw.bgw_name, BGW_MAXLEN,
"logical replication worker for subscription %u", subid);
+
snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = Int32GetDatum(slot);
+ if (is_subworker)
+ memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle));
+
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
/* Failed to start worker, so clean up the worker slot. */
@@ -421,11 +449,11 @@ retry:
(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
errmsg("out of background worker slots"),
errhint("You might need to increase max_worker_processes.")));
- return;
+ return false;
}
/* Now wait until it attaches. */
- WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
+ return WaitForReplicationWorkerAttach(worker, generation, bgw_handle);
}
/*
@@ -436,18 +464,27 @@ void
logicalrep_worker_stop(Oid subid, Oid relid)
{
LogicalRepWorker *worker;
- uint16 generation;
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
worker = logicalrep_worker_find(subid, relid, false);
- /* No worker, nothing to do. */
- if (!worker)
- {
- LWLockRelease(LogicalRepWorkerLock);
- return;
- }
+ if (worker)
+ logicalrep_worker_stop_internal(worker);
+
+ LWLockRelease(LogicalRepWorkerLock);
+}
+
+/*
+ * Workhorse for logicalrep_worker_stop() and logicalrep_worker_detach(). Stop
+ * the worker and wait for it to die.
+ */
+static void
+logicalrep_worker_stop_internal(LogicalRepWorker *worker)
+{
+ uint16 generation;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
/*
* Remember which generation was our worker so we can check if what we see
@@ -485,10 +522,7 @@ logicalrep_worker_stop(Oid subid, Oid relid)
* different, meaning that a different worker has taken the slot.
*/
if (!worker->in_use || worker->generation != generation)
- {
- LWLockRelease(LogicalRepWorkerLock);
return;
- }
/* Worker has assigned proc, so it has started. */
if (worker->proc)
@@ -522,8 +556,6 @@ logicalrep_worker_stop(Oid subid, Oid relid)
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
}
-
- LWLockRelease(LogicalRepWorkerLock);
}
/*
@@ -599,6 +631,29 @@ logicalrep_worker_attach(int slot)
static void
logicalrep_worker_detach(void)
{
+ /*
+ * If we are the main apply worker, stop all the apply background workers
+ * we started before.
+ */
+ if (!MyLogicalRepWorker->subworker)
+ {
+ List *workers;
+ ListCell *lc;
+
+ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+
+ workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true);
+ foreach(lc, workers)
+ {
+ LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc);
+
+ if (w->subworker)
+ logicalrep_worker_stop_internal(w);
+ }
+
+ LWLockRelease(LogicalRepWorkerLock);
+ }
+
/* Block concurrent access. */
LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
@@ -621,6 +676,7 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker)
worker->userid = InvalidOid;
worker->subid = InvalidOid;
worker->relid = InvalidOid;
+ worker->subworker = false;
}
/*
@@ -679,6 +735,30 @@ logicalrep_sync_worker_count(Oid subid)
return res;
}
+/*
+ * Count the number of registered (not necessarily running) apply background
+ * workers for a subscription.
+ */
+int
+logicalrep_apply_bgworker_count(Oid subid)
+{
+ int i;
+ int res = 0;
+
+ Assert(LWLockHeldByMe(LogicalRepWorkerLock));
+
+ /* Search for attached worker for a given subscription id. */
+ for (i = 0; i < max_logical_replication_workers; i++)
+ {
+ LogicalRepWorker *w = &LogicalRepCtx->workers[i];
+
+ if (w->subid == subid && w->subworker)
+ res++;
+ }
+
+ return res;
+}
+
/*
* ApplyLauncherShmemSize
* Compute space needed for replication launcher shared memory
@@ -868,7 +948,7 @@ ApplyLauncherMain(Datum main_arg)
wait_time = wal_retrieve_retry_interval;
logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
- sub->owner, InvalidOid);
+ sub->owner, InvalidOid, DSM_HANDLE_INVALID);
}
}
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index c72ad6b93d..2458e2fce9 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1075,12 +1075,21 @@ ReplicationOriginExitCleanup(int code, Datum arg)
* array doesn't have to be searched when calling
* replorigin_session_advance().
*
- * Obviously only one such cached origin can exist per process and the current
+ * Normally only one such cached origin can exist per process and the current
* cached value can only be set again after the previous value is torn down
* with replorigin_session_reset().
+ *
+ * However, if the function parameter 'must_acquire' is false, we allow the
+ * process to use the same slot already acquired by another process. It's safe
+ * because 1) The only caller (apply background workers) will maintain the
+ * commit order by allowing only one process to commit at a time, so no two
+ * workers will be operating on the same origin at the same time (see comments
+ * in logical/worker.c). 2) Even though we try to advance the session's origin
+ * concurrently, it's safe to do so as we change/advance the session_origin
+ * LSNs under replicate_state LWLock.
*/
void
-replorigin_session_setup(RepOriginId node)
+replorigin_session_setup(RepOriginId node, bool must_acquire)
{
static bool registered_cleanup;
int i;
@@ -1122,7 +1131,7 @@ replorigin_session_setup(RepOriginId node)
if (curstate->roident != node)
continue;
- else if (curstate->acquired_by != 0)
+ else if (curstate->acquired_by != 0 && must_acquire)
{
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
@@ -1153,7 +1162,14 @@ replorigin_session_setup(RepOriginId node)
Assert(session_replication_state->roident != InvalidRepOriginId);
- session_replication_state->acquired_by = MyProcPid;
+ if (must_acquire)
+ session_replication_state->acquired_by = MyProcPid;
+ else if (session_replication_state->acquired_by == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+ errmsg("apply background worker could not find replication state slot for replication origin with OID %u",
+ node),
+ errdetail("There is no replication state slot set by its main apply worker.")));
LWLockRelease(ReplicationOriginLock);
@@ -1337,7 +1353,7 @@ pg_replication_origin_session_setup(PG_FUNCTION_ARGS)
name = text_to_cstring((text *) DatumGetPointer(PG_GETARG_DATUM(0)));
origin = replorigin_by_name(name, false);
- replorigin_session_setup(origin);
+ replorigin_session_setup(origin, true);
replorigin_session_origin = origin;
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index ff8513e2d2..47bd811fb7 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -1163,31 +1163,56 @@ logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data)
/*
* Write STREAM ABORT to the output stream. Note that xid and subxid will be
* same for the top-level transaction abort.
+ *
+ * If write_abort_lsn is true, send the abort_lsn and abort_time fields,
+ * otherwise don't.
*/
void
logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid)
+ ReorderBufferTXN *txn, XLogRecPtr abort_lsn,
+ bool write_abort_lsn)
{
pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT);
- Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid));
+ Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(txn->xid));
/* transaction ID */
pq_sendint32(out, xid);
- pq_sendint32(out, subxid);
+ pq_sendint32(out, txn->xid);
+
+ if (write_abort_lsn)
+ {
+ pq_sendint64(out, abort_lsn);
+ pq_sendint64(out, txn->xact_time.abort_time);
+ }
}
/*
* Read STREAM ABORT from the output stream.
+ *
+ * If read_abort_lsn is true, try to read the abort_lsn and abort_time fields,
+ * otherwise don't.
*/
void
-logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid)
+logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn)
{
- Assert(xid && subxid);
+ Assert(abort_data);
- *xid = pq_getmsgint(in, 4);
- *subxid = pq_getmsgint(in, 4);
+ abort_data->xid = pq_getmsgint(in, 4);
+ abort_data->subxid = pq_getmsgint(in, 4);
+
+ if (read_abort_lsn)
+ {
+ abort_data->abort_lsn = pq_getmsgint64(in);
+ abort_data->abort_time = pq_getmsgint64(in);
+ }
+ else
+ {
+ abort_data->abort_lsn = InvalidXLogRecPtr;
+ abort_data->abort_time = 0;
+ }
}
/*
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 88a37fde72..8989328046 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -2826,7 +2826,8 @@ ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
* disk.
*/
void
-ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2837,6 +2838,8 @@ ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
{
@@ -2911,7 +2914,8 @@ ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid)
* to this xid might re-create the transaction incompletely.
*/
void
-ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
+ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
+ TimestampTz abort_time)
{
ReorderBufferTXN *txn;
@@ -2922,6 +2926,8 @@ ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn)
if (txn == NULL)
return;
+ txn->xact_time.abort_time = abort_time;
+
/* For streamed transactions notify the remote node about the abort. */
if (rbtxn_is_streamed(txn))
rb->stream_abort(rb, txn, lsn);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 670c6fcada..8ffba7e2e5 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -568,7 +568,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
MySubscription->oid,
MySubscription->name,
MyLogicalRepWorker->userid,
- rstate->relid);
+ rstate->relid,
+ DSM_HANDLE_INVALID);
hentry->last_start_time = now;
}
}
@@ -589,6 +590,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
void
process_syncing_tables(XLogRecPtr current_lsn)
{
+ if (MyLogicalRepWorker->subworker)
+ return;
+
if (am_tablesync_worker())
process_syncing_tables_for_sync(current_lsn);
else
@@ -1273,7 +1277,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
* time this tablesync was launched.
*/
originid = replorigin_by_name(originname, false);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
*origin_startpos = replorigin_session_get_progress(false);
@@ -1384,7 +1388,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
true /* go backward */ , true /* WAL log */ );
UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
}
else
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 5f8c541763..2aa7797628 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -22,8 +22,28 @@
* STREAMED TRANSACTIONS
* ---------------------
* Streamed transactions (large transactions exceeding a memory limit on the
- * upstream) are not applied immediately, but instead, the data is written
- * to temporary files and then applied at once when the final commit arrives.
+ * upstream) are applied using one of two approaches.
+ *
+ * 1) Separate background workers
+ *
+ * If streaming = parallel, we assign a new apply background worker (if
+ * available) as soon as the xact's first stream is received. The main apply
+ * worker will send changes to this new worker via shared memory. We keep this
+ * worker assigned till the transaction commit is received and also wait for
+ * the worker to finish at commit. This preserves commit ordering and avoids
+ * file I/O in most cases. We still need to spill to a file if there is no
+ * worker available. It is important to maintain commit order to avoid failures
+ * due to (a) transaction dependencies, say if we insert a row in the first
+ * transaction and update it in the second transaction then allowing to apply
+ * both in parallel can lead to failure in the update. (b) deadlocks, allowing
+ * transactions that update the same set of rows/tables in opposite order to be
+ * applied in parallel can lead to deadlocks.
+ *
+ * 2) Write to temporary files and apply when the final commit arrives
+ *
+ * If no worker is available to handle streamed transaction, the data is
+ * written to temporary files and then applied at once when the final commit
+ * arrives.
*
* Unlike the regular (non-streamed) case, handling streamed transactions has
* to handle aborts of both the toplevel transaction and subtransactions. This
@@ -219,20 +239,8 @@ typedef struct ApplyExecutionData
PartitionTupleRouting *proute; /* partition routing info */
} ApplyExecutionData;
-/* Struct for saving and restoring apply errcontext information */
-typedef struct ApplyErrorCallbackArg
-{
- LogicalRepMsgType command; /* 0 if invalid */
- LogicalRepRelMapEntry *rel;
-
- /* Remote node information */
- int remote_attnum; /* -1 if invalid */
- TransactionId remote_xid;
- XLogRecPtr finish_lsn;
- char *origin_name;
-} ApplyErrorCallbackArg;
-
-static ApplyErrorCallbackArg apply_error_callback_arg =
+/* errcontext tracker */
+ApplyErrorCallbackArg apply_error_callback_arg =
{
.command = 0,
.rel = NULL,
@@ -242,7 +250,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg =
.origin_name = NULL,
};
-static MemoryContext ApplyMessageContext = NULL;
+MemoryContext ApplyMessageContext = NULL;
MemoryContext ApplyContext = NULL;
/* per stream context for streaming transactions */
@@ -251,27 +259,39 @@ static MemoryContext LogicalStreamingContext = NULL;
WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
Subscription *MySubscription = NULL;
-static bool MySubscriptionValid = false;
+bool MySubscriptionValid = false;
bool in_remote_transaction = false;
static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
/* fields valid only when processing streamed transaction */
-static bool in_streamed_transaction = false;
+bool in_streamed_transaction = false;
+
+TransactionId stream_xid = InvalidTransactionId;
+static ApplyBgworkerState *stream_apply_worker = NULL;
-static TransactionId stream_xid = InvalidTransactionId;
+/* Check if we are applying the transaction in an apply background worker */
+#define apply_bgworker_active() (in_streamed_transaction && stream_apply_worker != NULL)
+
+/*
+ * The number of changes during one streaming block (only for apply background
+ * workers)
+ */
+static uint32 nchanges = 0;
/*
* We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
* the subscription if the remote transaction's finish LSN matches the subskiplsn.
* Once we start skipping changes, we don't stop it until we skip all changes of
* the transaction even if pg_subscription is updated and MySubscription->skiplsn
- * gets changed or reset during that. Also, in streaming transaction cases, we
- * don't skip receiving and spooling the changes since we decide whether or not
+ * gets changed or reset during that. Also, in streaming transaction cases (streaming = on),
+ * we don't skip receiving and spooling the changes since we decide whether or not
* to skip applying the changes when starting to apply changes. The subskiplsn is
* cleared after successfully skipping the transaction or applying non-empty
* transaction. The latter prevents the mistakenly specified subskiplsn from
- * being left.
+ * being left. Note that we cannot skip the streaming transactions when using
+ * apply background workers because we cannot get the finish LSN before
+ * applying the changes.
*/
static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr;
#define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn)))
@@ -324,9 +344,6 @@ static void maybe_reread_subscription(void);
static void DisableSubscriptionAndExit(void);
-/* prototype needed because of stream_commit */
-static void apply_dispatch(StringInfo s);
-
static void apply_handle_commit_internal(LogicalRepCommitData *commit_data);
static void apply_handle_insert_internal(ApplyExecutionData *edata,
ResultRelInfo *relinfo,
@@ -359,7 +376,6 @@ static void stop_skipping_changes(void);
static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
/* Functions for apply error callback */
-static void apply_error_callback(void *arg);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
@@ -426,40 +442,85 @@ end_replication_step(void)
}
/*
- * Handle streamed transactions.
+ * Handle streamed transactions for both the main apply worker and the apply
+ * background workers.
+ *
+ * In streaming case (receiving a block of streamed transaction), for
+ * SUBSTREAM_ON mode, simply redirect it to a file for the proper toplevel
+ * transaction, and for SUBSTREAM_PARALLEL mode, send the changes to apply
+ * background workers (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes
+ * will also be applied in main apply worker).
*
- * If in streaming mode (receiving a block of streamed transaction), we
- * simply redirect it to a file for the proper toplevel transaction.
+ * For non-streamed transactions, returns false;
+ * For streamed transactions, returns true if in main apply worker, false
+ * otherwise.
*
- * Returns true for streamed transactions, false otherwise (regular mode).
+ * Exception: When the main apply worker is applying streaming transactions in
+ * parallel mode (e.g. when addressing LOGICAL_REP_MSG_RELATION or
+ * LOGICAL_REP_MSG_TYPE changes), then return false.
*/
static bool
handle_streamed_transaction(LogicalRepMsgType action, StringInfo s)
{
- TransactionId xid;
+ TransactionId current_xid = InvalidTransactionId;
- /* not in streaming mode */
- if (!in_streamed_transaction)
+ /* Not in streaming mode */
+ if (!(in_streamed_transaction || am_apply_bgworker()))
return false;
- Assert(stream_fd != NULL);
Assert(TransactionIdIsValid(stream_xid));
/*
* We should have received XID of the subxact as the first part of the
* message, so extract it.
*/
- xid = pq_getmsgint(s, 4);
+ current_xid = pq_getmsgint(s, 4);
- if (!TransactionIdIsValid(xid))
+ if (!TransactionIdIsValid(current_xid))
ereport(ERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("invalid transaction ID in streamed replication transaction")));
- /* Add the new subxact to the array (unless already there). */
- subxact_info_add(xid);
+ if (am_apply_bgworker())
+ {
+ /* Define a savepoint for a subxact if needed. */
+ apply_bgworker_subxact_info_add(current_xid);
+
+ return false;
+ }
+
+ if (apply_bgworker_active())
+ {
+ /*
+ * This is the main apply worker, but there is an apply background
+ * worker, so apply the changes of this transaction in that background
+ * worker. Pass the data to the worker.
+ */
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
+ nchanges += 1;
+
+ /*
+ * XXX The publisher side doesn't always send relation/type update
+ * messages after the streaming transaction, so also update the
+ * relation/type in main apply worker here. See function
+ * cleanup_rel_sync_cache.
+ */
+ if (action == LOGICAL_REP_MSG_RELATION ||
+ action == LOGICAL_REP_MSG_TYPE)
+ return false;
+
+ return true;
+ }
+
+ /*
+ * This is the main apply worker, but there is no apply background worker,
+ * so write to temporary files and apply when the final commit arrives.
+ *
+ * Add the new subxact to the array (unless already there).
+ */
+ subxact_info_add(current_xid);
- /* write the change to the current file */
+ /* Write the change to the current file */
stream_write_change(action, s);
return true;
@@ -844,6 +905,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -898,7 +962,9 @@ apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data)
* BeginTransactionBlock is necessary to balance the EndTransactionBlock
* called within the PrepareTransactionBlock below.
*/
- BeginTransactionBlock();
+ if (!IsTransactionBlock())
+ BeginTransactionBlock();
+
CommitTransactionCommand(); /* Completes the preceding Begin command. */
/*
@@ -950,6 +1016,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1064,10 +1133,6 @@ apply_handle_rollback_prepared(StringInfo s)
/*
* Handle STREAM PREPARE.
- *
- * Logic is in two parts:
- * 1. Replay all the spooled operations
- * 2. Mark the transaction as prepared
*/
static void
apply_handle_stream_prepare(StringInfo s)
@@ -1088,24 +1153,78 @@ apply_handle_stream_prepare(StringInfo s)
logicalrep_read_stream_prepare(s, &prepare_data);
set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn);
- elog(DEBUG1, "received prepare for streamed transaction %u", prepare_data.xid);
+ if (am_apply_bgworker())
+ {
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
- /* Replay all the spooled operations. */
- apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+ CommitTransactionCommand();
- /* Mark the transaction as prepared. */
- apply_handle_prepare_internal(&prepare_data);
+ pgstat_report_stat(false);
- CommitTransactionCommand();
+ list_free(subxactlist);
+ subxactlist = NIL;
- pgstat_report_stat(false);
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(prepare_data.xid);
- store_flush_position(prepare_data.end_lsn);
+ elog(DEBUG1, "received prepare for streamed transaction %u",
+ prepare_data.xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM PREPARE message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+
+ pgstat_report_stat(false);
+ store_flush_position(prepare_data.end_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn);
+
+ /* Mark the transaction as prepared. */
+ apply_handle_prepare_internal(&prepare_data);
+
+ CommitTransactionCommand();
+
+ pgstat_report_stat(false);
+
+ store_flush_position(prepare_data.end_lsn);
+
+ in_remote_transaction = false;
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ }
+ }
in_remote_transaction = false;
+ stream_apply_worker = NULL;
- /* unlink the files with serialized changes and subxact info. */
- stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1155,15 +1274,6 @@ apply_handle_stream_start(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("duplicate STREAM START message")));
- /*
- * Start a transaction on stream start, this transaction will be committed
- * on the stream stop unless it is a tablesync worker in which case it
- * will be committed after processing all the messages. We need the
- * transaction for handling the buffile, used for serializing the
- * streaming data and subxact info.
- */
- begin_replication_step();
-
/* notify handle methods we're processing a remote transaction */
in_streamed_transaction = true;
@@ -1177,36 +1287,93 @@ apply_handle_stream_start(StringInfo s)
set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr);
- /*
- * Initialize the worker's stream_fileset if we haven't yet. This will be
- * used for the entire duration of the worker so create it in a permanent
- * context. We create this on the very first streaming message from any
- * transaction and then use it for this and other streaming transactions.
- * Now, we could create a fileset at the start of the worker as well but
- * then we won't be sure that it will ever be used.
- */
- if (MyLogicalRepWorker->stream_fileset == NULL)
+ if (am_apply_bgworker())
{
- MemoryContext oldctx;
-
- oldctx = MemoryContextSwitchTo(ApplyContext);
+ /*
+ * Make sure the handle apply_dispatch methods are aware we're in a
+ * remote transaction.
+ */
+ in_remote_transaction = true;
- MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
- FileSetInit(MyLogicalRepWorker->stream_fileset);
+ /* Begin the transaction. */
+ AcceptInvalidationMessages();
+ maybe_reread_subscription();
- MemoryContextSwitchTo(oldctx);
+ StartTransactionCommand();
+ BeginTransactionBlock();
+ CommitTransactionCommand();
}
+ else
+ {
+ /*
+ * This is the main apply worker. Check if there is any free apply
+ * background worker we can use to process this transaction.
+ */
+ if (first_segment)
+ stream_apply_worker = apply_bgworker_start(stream_xid);
+ else
+ stream_apply_worker = apply_bgworker_find(stream_xid);
- /* open the spool file for this transaction */
- stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+ if (stream_apply_worker)
+ {
+ /*
+ * If we have found a free worker or if we are already applying this
+ * transaction in an apply background worker, then we pass the data to
+ * that worker.
+ */
+ if (first_segment)
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
- /* if this is not the first segment, open existing subxact file */
- if (!first_segment)
- subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+ nchanges = 0;
+ elog(DEBUG1, "starting streaming of xid %u", stream_xid);
+ }
+ else
+ {
+ /*
+ * Since no apply background worker is available for the first
+ * stream start, serialize all the changes of the transaction.
+ *
+ * Start a transaction on stream start, this transaction will be
+ * committed on the stream stop unless it is a tablesync worker in
+ * which case it will be committed after processing all the
+ * messages. We need the transaction for handling the buffile,
+ * used for serializing the streaming data and subxact info.
+ */
+ begin_replication_step();
- pgstat_report_activity(STATE_RUNNING, NULL);
+ /*
+ * Initialize the worker's stream_fileset if we haven't yet. This will
+ * be used for the entire duration of the worker so create it in a
+ * permanent context. We create this on the very first streaming
+ * message from any transaction and then use it for this and other
+ * streaming transactions. Now, we could create a fileset at the start
+ * of the worker as well but then we won't be sure that it will ever
+ * be used.
+ */
+ if (MyLogicalRepWorker->stream_fileset == NULL)
+ {
+ MemoryContext oldctx;
- end_replication_step();
+ oldctx = MemoryContextSwitchTo(ApplyContext);
+
+ MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
+ FileSetInit(MyLogicalRepWorker->stream_fileset);
+
+ MemoryContextSwitchTo(oldctx);
+ }
+
+ /* Open the spool file for this transaction. */
+ stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment);
+
+ /* If this is not the first segment, open existing subxact file. */
+ if (!first_segment)
+ subxact_info_read(MyLogicalRepWorker->subid, stream_xid);
+
+ end_replication_step();
+ }
+ }
+
+ pgstat_report_activity(STATE_RUNNING, NULL);
}
/*
@@ -1220,53 +1387,52 @@ apply_handle_stream_stop(StringInfo s)
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg_internal("STREAM STOP message without STREAM START")));
- /*
- * Close the file with serialized changes, and serialize information about
- * subxacts for the toplevel transaction.
- */
- subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
- stream_close_file();
+ if (apply_bgworker_active())
+ {
+ char action = LOGICAL_REP_MSG_STREAM_STOP;
- /* We must be in a valid transaction state */
- Assert(IsTransactionState());
+ apply_bgworker_send_data(stream_apply_worker, 1, &action);
- /* Commit the per-stream transaction */
- CommitTransactionCommand();
+ elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
+ }
+ else
+ {
+ /*
+ * Close the file with serialized changes, and serialize information
+ * about subxacts for the toplevel transaction.
+ */
+ subxact_info_write(MyLogicalRepWorker->subid, stream_xid);
+ stream_close_file();
- in_streamed_transaction = false;
+ /* We must be in a valid transaction state */
+ Assert(IsTransactionState());
- /* Reset per-stream context */
- MemoryContextReset(LogicalStreamingContext);
+ /* Commit the per-stream transaction */
+ CommitTransactionCommand();
+
+ /* Reset per-stream context */
+ MemoryContextReset(LogicalStreamingContext);
+ }
+
+ in_streamed_transaction = false;
+ stream_apply_worker = NULL;
pgstat_report_activity(STATE_IDLE, NULL);
reset_apply_error_context_info();
}
/*
- * Handle STREAM abort message.
+ * Handle STREAM ABORT message when the transaction was spilled to disk.
*/
static void
-apply_handle_stream_abort(StringInfo s)
+serialize_stream_abort(TransactionId xid, TransactionId subxid)
{
- TransactionId xid;
- TransactionId subxid;
-
- if (in_streamed_transaction)
- ereport(ERROR,
- (errcode(ERRCODE_PROTOCOL_VIOLATION),
- errmsg_internal("STREAM ABORT message without STREAM STOP")));
-
- logicalrep_read_stream_abort(s, &xid, &subxid);
-
/*
* If the two XIDs are the same, it's in fact abort of toplevel xact, so
* just delete the files with serialized info.
*/
if (xid == subxid)
- {
- set_apply_error_context_xact(xid, InvalidXLogRecPtr);
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
- }
else
{
/*
@@ -1290,8 +1456,6 @@ apply_handle_stream_abort(StringInfo s)
bool found = false;
char path[MAXPGPATH];
- set_apply_error_context_xact(subxid, InvalidXLogRecPtr);
-
subidx = -1;
begin_replication_step();
subxact_info_read(MyLogicalRepWorker->subid, xid);
@@ -1316,7 +1480,6 @@ apply_handle_stream_abort(StringInfo s)
cleanup_subxact_info();
end_replication_step();
CommitTransactionCommand();
- reset_apply_error_context_info();
return;
}
@@ -1339,6 +1502,143 @@ apply_handle_stream_abort(StringInfo s)
end_replication_step();
CommitTransactionCommand();
}
+}
+
+/*
+ * Handle STREAM ABORT message.
+ */
+static void
+apply_handle_stream_abort(StringInfo s)
+{
+ TransactionId xid;
+ TransactionId subxid;
+ LogicalRepStreamAbortData abort_data;
+ bool read_abort_lsn = false;
+
+ if (in_streamed_transaction)
+ ereport(ERROR,
+ (errcode(ERRCODE_PROTOCOL_VIOLATION),
+ errmsg_internal("STREAM ABORT message without STREAM STOP")));
+
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelShared->server_version >=
+ LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM;
+
+ logicalrep_read_stream_abort(s, &abort_data, read_abort_lsn);
+
+ xid = abort_data.xid;
+ subxid = abort_data.subxid;
+
+ set_apply_error_context_xact(subxid, abort_data.abort_lsn);
+
+ if (am_apply_bgworker())
+ {
+ elog(DEBUG1, "[Apply BGW #%u] aborting current transaction xid=%u, subxid=%u",
+ MyParallelShared->n, GetCurrentTransactionIdIfAny(),
+ GetCurrentSubTransactionId());
+
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ if (read_abort_lsn)
+ {
+ replorigin_session_origin_lsn = abort_data.abort_lsn;
+ replorigin_session_origin_timestamp = abort_data.abort_time;
+ }
+
+ /*
+ * If the two XIDs are the same, it's in fact abort of toplevel xact,
+ * so just free the subxactlist.
+ */
+ if (subxid == xid)
+ {
+ AbortCurrentTransaction();
+
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
+
+ in_remote_transaction = false;
+ pgstat_report_activity(STATE_IDLE, NULL);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /*
+ * OK, so it's a subxact. Rollback to the savepoint.
+ *
+ * We also need to read the subxactlist, determine the offset
+ * tracked for the subxact, and truncate the list.
+ */
+ int i;
+ bool found = false;
+ char spname[MAXPGPATH];
+
+ snprintf(spname, MAXPGPATH, "savepoint_for_xid_%u", subxid);
+
+ elog(DEBUG1, "[Apply BGW #%u] rolling back to savepoint %s",
+ MyParallelShared->n, spname);
+
+ for (i = list_length(subxactlist) - 1; i >= 0; i--)
+ {
+ xid = (TransactionId) list_nth_int(subxactlist, i);
+ if (xid == subxid)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if (found)
+ {
+ RollbackToSavepoint(spname);
+ CommitTransactionCommand();
+ subxactlist = list_truncate(subxactlist, i + 1);
+ }
+
+ pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
+ }
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM ABORT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ if (subxid == xid)
+ {
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
+ apply_bgworker_free(wstate);
+ }
+ }
+ else
+ {
+ /*
+ * We are in main apply worker and the transaction has been
+ * serialized to file.
+ */
+ serialize_stream_abort(xid, subxid);
+ }
+ }
reset_apply_error_context_info();
}
@@ -1468,8 +1768,8 @@ apply_spooled_messages(TransactionId xid, XLogRecPtr lsn)
static void
apply_handle_stream_commit(StringInfo s)
{
- TransactionId xid;
LogicalRepCommitData commit_data;
+ TransactionId xid;
if (in_streamed_transaction)
ereport(ERROR,
@@ -1479,14 +1779,81 @@ apply_handle_stream_commit(StringInfo s)
xid = logicalrep_read_stream_commit(s, &commit_data);
set_apply_error_context_xact(xid, commit_data.commit_lsn);
- elog(DEBUG1, "received commit for streamed transaction %u", xid);
+ if (am_apply_bgworker())
+ {
+ /*
+ * Update origin state so we can restart streaming from correct
+ * position in case of crash.
+ */
+ replorigin_session_origin_lsn = commit_data.end_lsn;
+ replorigin_session_origin_timestamp = commit_data.committime;
- apply_spooled_messages(xid, commit_data.commit_lsn);
+ CommitTransactionCommand();
+ EndTransactionBlock(false);
+ CommitTransactionCommand();
- apply_handle_commit_internal(&commit_data);
+ in_remote_transaction = false;
+
+ pgstat_report_stat(false);
+
+ list_free(subxactlist);
+ subxactlist = NIL;
+
+ apply_bgworker_set_status(APPLY_BGWORKER_FINISHED);
+ }
+ else
+ {
+ /* This is the main apply worker. */
+ ApplyBgworkerState *wstate = apply_bgworker_find(xid);
+
+ elog(DEBUG1, "received commit for streamed transaction %u", xid);
+
+ /*
+ * Check if we are processing this transaction in an apply background
+ * worker and if so, send the changes to that worker.
+ */
+ if (wstate)
+ {
+ /* Send STREAM COMMIT message to the apply background worker. */
+ apply_bgworker_send_data(wstate, s->len, s->data);
+
+ /*
+ * After sending the data to the apply background worker, wait for
+ * that worker to finish. This is necessary to maintain commit
+ * order which avoids failures due to transaction dependencies and
+ * deadlocks.
+ */
+ apply_bgworker_wait_for(wstate, APPLY_BGWORKER_FINISHED);
- /* unlink the files with serialized changes and subxact info */
- stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ pgstat_report_stat(false);
+ store_flush_position(commit_data.end_lsn);
+ stop_skipping_changes();
+
+ apply_bgworker_free(wstate);
+
+ /*
+ * The transaction is either non-empty or skipped, so we clear the
+ * subskiplsn.
+ */
+ clear_subscription_skip_lsn(commit_data.commit_lsn);
+ }
+ else
+ {
+ /*
+ * The transaction has been serialized to file, so replay all the
+ * spooled operations.
+ */
+ apply_spooled_messages(xid, commit_data.commit_lsn);
+
+ apply_handle_commit_internal(&commit_data);
+
+ /* Unlink the files with serialized changes and subxact info. */
+ stream_cleanup_files(MyLogicalRepWorker->subid, xid);
+ }
+ }
+
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(commit_data.end_lsn);
@@ -2467,7 +2834,7 @@ apply_handle_truncate(StringInfo s)
/*
* Logical replication protocol message dispatcher.
*/
-static void
+void
apply_dispatch(StringInfo s)
{
LogicalRepMsgType action = pq_getmsgbyte(s);
@@ -2636,6 +3003,10 @@ store_flush_position(XLogRecPtr remote_lsn)
{
FlushPosition *flushpos;
+ /* Skip if not the main apply worker */
+ if (am_apply_bgworker())
+ return;
+
/* Need to do this in permanent context */
MemoryContextSwitchTo(ApplyContext);
@@ -2650,7 +3021,7 @@ store_flush_position(XLogRecPtr remote_lsn)
/* Update statistics of the worker. */
-static void
+void
UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply)
{
MyLogicalRepWorker->last_lsn = last_lsn;
@@ -2812,6 +3183,9 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
AcceptInvalidationMessages();
maybe_reread_subscription();
+ /* Check the status of apply background worker if any. */
+ apply_bgworker_check_status();
+
/* Process any table synchronization changes. */
process_syncing_tables(last_received);
}
@@ -3114,7 +3488,7 @@ maybe_reread_subscription(void)
/*
* Callback from subscription syscache invalidation.
*/
-static void
+void
subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue)
{
MySubscriptionValid = false;
@@ -3710,7 +4084,7 @@ ApplyWorkerMain(Datum main_arg)
originid = replorigin_by_name(originname, true);
if (!OidIsValid(originid))
originid = replorigin_create(originname);
- replorigin_session_setup(originid);
+ replorigin_session_setup(originid, true);
replorigin_session_origin = originid;
origin_startpos = replorigin_session_get_progress(false);
CommitTransactionCommand();
@@ -3751,13 +4125,14 @@ ApplyWorkerMain(Datum main_arg)
server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
options.proto.logical.proto_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
LOGICALREP_PROTO_VERSION_NUM;
options.proto.logical.publication_names = MySubscription->publications;
options.proto.logical.binary = MySubscription->binary;
- options.proto.logical.streaming = MySubscription->stream;
+ options.proto.logical.streaming = (MySubscription->stream != SUBSTREAM_OFF);
options.proto.logical.twophase = false;
options.proto.logical.origin = pstrdup(MySubscription->origin);
@@ -3916,7 +4291,8 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
XLogRecPtr myskiplsn = MySubscription->skiplsn;
bool started_tx = false;
- if (likely(XLogRecPtrIsInvalid(myskiplsn)))
+ if (likely(XLogRecPtrIsInvalid(myskiplsn)) ||
+ am_apply_bgworker())
return;
if (!IsTransactionState())
@@ -3988,7 +4364,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
}
/* Error callback to give more context info about the change being applied */
-static void
+void
apply_error_callback(void *arg)
{
ApplyErrorCallbackArg *errarg = &apply_error_callback_arg;
@@ -4016,23 +4392,47 @@ apply_error_callback(void *arg)
errarg->remote_xid,
LSN_FORMAT_ARGS(errarg->finish_lsn));
}
- else if (errarg->remote_attnum < 0)
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
else
- errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
- errarg->origin_name,
- logicalrep_message_type(errarg->command),
- errarg->rel->remoterel.nspname,
- errarg->rel->remoterel.relname,
- errarg->rel->remoterel.attnames[errarg->remote_attnum],
- errarg->remote_xid,
- LSN_FORMAT_ARGS(errarg->finish_lsn));
+ {
+ if (errarg->remote_attnum < 0)
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ else
+ {
+ if (XLogRecPtrIsInvalid(errarg->finish_lsn))
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid);
+ else
+ errcontext("processing remote data for replication origin \"%s\" during \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u finished at %X/%X",
+ errarg->origin_name,
+ logicalrep_message_type(errarg->command),
+ errarg->rel->remoterel.nspname,
+ errarg->rel->remoterel.relname,
+ errarg->rel->remoterel.attnames[errarg->remote_attnum],
+ errarg->remote_xid,
+ LSN_FORMAT_ARGS(errarg->finish_lsn));
+ }
+ }
}
/* Set transaction information of apply error callback */
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index a3c1ba8a40..f9e388ada1 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -1843,6 +1843,9 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
XLogRecPtr abort_lsn)
{
ReorderBufferTXN *toptxn;
+ PGOutputData *data = (PGOutputData *) ctx->output_plugin_private;
+ bool write_abort_lsn = (data->protocol_version >=
+ LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM);
/*
* The abort should happen outside streaming block, even for streamed
@@ -1856,7 +1859,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx,
Assert(rbtxn_is_streamed(toptxn));
OutputPluginPrepareWrite(ctx, true);
- logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid);
+ logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn, abort_lsn,
+ write_abort_lsn);
OutputPluginWrite(ctx, true);
cleanup_rel_sync_cache(toptxn->xid, false);
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index da57a93034..2e146fe087 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -388,6 +388,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT:
event_name = "HashGrowBucketsReinsert";
break;
+ case WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE:
+ event_name = "LogicalApplyWorkerStateChange";
+ break;
case WAIT_EVENT_LOGICAL_SYNC_DATA:
event_name = "LogicalSyncData";
break;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index af4a1c3068..a40ddf847a 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3220,6 +3220,18 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_apply_bgworkers_per_subscription",
+ PGC_SIGHUP,
+ REPLICATION_SUBSCRIBERS,
+ gettext_noop("Maximum number of apply background workers per subscription."),
+ NULL,
+ },
+ &max_apply_bgworkers_per_subscription,
+ 2, 0, MAX_BACKENDS,
+ NULL, NULL, NULL
+ },
+
{
{"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Sets the amount of time to wait before forcing "
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..ad18710af4 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,7 @@
#max_logical_replication_workers = 4 # taken from max_worker_processes
# (change requires restart)
#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers
+#max_apply_bgworkers_per_subscription = 2 # taken from max_logical_replication_workers
#------------------------------------------------------------------------------
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f9c51d1e67..b894cca929 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4457,7 +4457,7 @@ getSubscriptions(Archive *fout)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query, " s.substream,\n");
else
- appendPQExpBufferStr(query, " false AS substream,\n");
+ appendPQExpBufferStr(query, " 'f' AS substream,\n");
if (fout->remoteVersion >= 150000)
appendPQExpBufferStr(query,
@@ -4594,8 +4594,10 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo)
if (strcmp(subinfo->subbinary, "t") == 0)
appendPQExpBufferStr(query, ", binary = true");
- if (strcmp(subinfo->substream, "f") != 0)
+ if (strcmp(subinfo->substream, "t") == 0)
appendPQExpBufferStr(query, ", streaming = on");
+ else if (strcmp(subinfo->substream, "p") == 0)
+ appendPQExpBufferStr(query, ", streaming = parallel");
if (strcmp(subinfo->subtwophasestate, two_phase_disabled) != 0)
appendPQExpBufferStr(query, ", two_phase = on");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index c9a3026b28..71ad03a934 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -80,7 +80,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subbinary; /* True if the subscription wants the
* publisher to send data in binary */
- bool substream; /* Stream in-progress transactions. */
+ char substream; /* Stream in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char subtwophasestate; /* Stream two-phase transactions */
@@ -124,7 +125,8 @@ typedef struct Subscription
bool enabled; /* Indicates if the subscription is enabled */
bool binary; /* Indicates if the subscription wants data in
* binary format */
- bool stream; /* Allow streaming in-progress transactions. */
+ char stream; /* Allow streaming in-progress transactions.
+ * See SUBSTREAM_xxx constants. */
char twophasestate; /* Allow streaming two-phase transactions */
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
@@ -137,6 +139,21 @@ typedef struct Subscription
* specified origin */
} Subscription;
+/* Disallow streaming in-progress transactions. */
+#define SUBSTREAM_OFF 'f'
+
+/*
+ * Streaming in-progress transactions are written to a temporary file and
+ * applied only after the transaction is committed on upstream.
+ */
+#define SUBSTREAM_ON 't'
+
+/*
+ * Streaming in-progress transactions are applied immediately via a background
+ * worker.
+ */
+#define SUBSTREAM_PARALLEL 'p'
+
extern Subscription *GetSubscription(Oid subid, bool missing_ok);
extern void FreeSubscription(Subscription *sub);
extern void DisableSubscription(Oid subid);
diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h
index f1e2821e25..ac8ef94381 100644
--- a/src/include/replication/logicallauncher.h
+++ b/src/include/replication/logicallauncher.h
@@ -14,6 +14,7 @@
extern PGDLLIMPORT int max_logical_replication_workers;
extern PGDLLIMPORT int max_sync_workers_per_subscription;
+extern PGDLLIMPORT int max_apply_bgworkers_per_subscription;
extern void ApplyLauncherRegister(void);
extern void ApplyLauncherMain(Datum main_arg);
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index a771ab8ff3..eb0fd24fd8 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -32,12 +32,17 @@
*
* LOGICALREP_PROTO_TWOPHASE_VERSION_NUM is the minimum protocol version with
* support for two-phase commit decoding (at prepare time). Introduced in PG15.
+ *
+ * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version
+ * with support for streaming large transactions using apply background
+ * workers. Introduced in PG16.
*/
#define LOGICALREP_PROTO_MIN_VERSION_NUM 1
#define LOGICALREP_PROTO_VERSION_NUM 1
#define LOGICALREP_PROTO_STREAM_VERSION_NUM 2
#define LOGICALREP_PROTO_TWOPHASE_VERSION_NUM 3
-#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_TWOPHASE_VERSION_NUM
+#define LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM 4
+#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM
/*
* Logical message types
@@ -175,6 +180,17 @@ typedef struct LogicalRepRollbackPreparedTxnData
char gid[GIDSIZE];
} LogicalRepRollbackPreparedTxnData;
+/*
+ * Transaction protocol information for stream abort.
+ */
+typedef struct LogicalRepStreamAbortData
+{
+ TransactionId xid;
+ TransactionId subxid;
+ XLogRecPtr abort_lsn;
+ TimestampTz abort_time;
+} LogicalRepStreamAbortData;
+
extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn);
extern void logicalrep_read_begin(StringInfo in,
LogicalRepBeginData *begin_data);
@@ -246,9 +262,12 @@ extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn
extern TransactionId logicalrep_read_stream_commit(StringInfo out,
LogicalRepCommitData *commit_data);
extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid,
- TransactionId subxid);
-extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid,
- TransactionId *subxid);
+ ReorderBufferTXN *txn,
+ XLogRecPtr abort_lsn,
+ bool write_abort_lsn);
+extern void logicalrep_read_stream_abort(StringInfo in,
+ LogicalRepStreamAbortData *abort_data,
+ bool read_abort_lsn);
extern char *logicalrep_message_type(LogicalRepMsgType action);
#endif /* LOGICAL_PROTO_H */
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index cd1b6e8afc..6a1af7f13c 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -13,6 +13,7 @@
#define LOGICALWORKER_H
extern void ApplyWorkerMain(Datum main_arg);
+extern void ApplyBgworkerMain(Datum main_arg);
extern bool IsLogicalWorker(void);
diff --git a/src/include/replication/origin.h b/src/include/replication/origin.h
index 2d1b5e5c28..c7389b40a7 100644
--- a/src/include/replication/origin.h
+++ b/src/include/replication/origin.h
@@ -53,7 +53,7 @@ extern XLogRecPtr replorigin_get_progress(RepOriginId node, bool flush);
extern void replorigin_session_advance(XLogRecPtr remote_commit,
XLogRecPtr local_commit);
-extern void replorigin_session_setup(RepOriginId node);
+extern void replorigin_session_setup(RepOriginId node, bool must_acquire);
extern void replorigin_session_reset(void);
extern XLogRecPtr replorigin_session_get_progress(bool flush);
diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index d109d0baed..d2a80d79e5 100644
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -301,6 +301,7 @@ typedef struct ReorderBufferTXN
{
TimestampTz commit_time;
TimestampTz prepare_time;
+ TimestampTz abort_time;
} xact_time;
/*
@@ -647,9 +648,11 @@ extern void ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid,
extern void ReorderBufferAssignChild(ReorderBuffer *, TransactionId, TransactionId, XLogRecPtr commit_lsn);
extern void ReorderBufferCommitChild(ReorderBuffer *, TransactionId, TransactionId,
XLogRecPtr commit_lsn, XLogRecPtr end_lsn);
-extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferAbort(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferAbortOld(ReorderBuffer *, TransactionId xid);
-extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
+extern void ReorderBufferForget(ReorderBuffer *, TransactionId, XLogRecPtr lsn,
+ TimestampTz abort_time);
extern void ReorderBufferInvalidate(ReorderBuffer *, TransactionId, XLogRecPtr lsn);
extern void ReorderBufferSetBaseSnapshot(ReorderBuffer *, TransactionId, XLogRecPtr lsn, struct SnapshotData *snap);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..a3560d4904 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -17,8 +17,11 @@
#include "access/xlogdefs.h"
#include "catalog/pg_subscription.h"
#include "datatype/timestamp.h"
+#include "replication/logicalrelation.h"
#include "storage/fileset.h"
#include "storage/lock.h"
+#include "storage/shm_mq.h"
+#include "storage/shm_toc.h"
#include "storage/spin.h"
@@ -60,6 +63,9 @@ typedef struct LogicalRepWorker
*/
FileSet *stream_fileset;
+ /* Indicates if this slot is used for an apply background worker. */
+ bool subworker;
+
/* Stats. */
XLogRecPtr last_lsn;
TimestampTz last_send_time;
@@ -68,8 +74,68 @@ typedef struct LogicalRepWorker
TimestampTz reply_time;
} LogicalRepWorker;
+/* Struct for saving and restoring apply errcontext information */
+typedef struct ApplyErrorCallbackArg
+{
+ LogicalRepMsgType command; /* 0 if invalid */
+ LogicalRepRelMapEntry *rel;
+
+ /* Remote node information */
+ int remote_attnum; /* -1 if invalid */
+ TransactionId remote_xid;
+ XLogRecPtr finish_lsn;
+ char *origin_name;
+} ApplyErrorCallbackArg;
+
+/*
+ * Status of apply background worker.
+ */
+typedef enum ApplyBgworkerStatus
+{
+ APPLY_BGWORKER_BUSY = 0, /* assigned to a transaction */
+ APPLY_BGWORKER_FINISHED, /* transaction is completed */
+ APPLY_BGWORKER_EXIT /* exit */
+} ApplyBgworkerStatus;
+
+/*
+ * Struct for sharing information between apply main and apply background
+ * workers.
+ */
+typedef struct ApplyBgworkerShared
+{
+ slock_t mutex;
+
+ /* Status of apply background worker. */
+ ApplyBgworkerStatus status;
+
+ /* server version of publisher. */
+ uint32 server_version;
+
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
+
+/*
+ * Struct for maintaining an apply background worker.
+ */
+typedef struct ApplyBgworkerState
+{
+ shm_mq_handle *mq_handle;
+ dsm_segment *dsm_seg;
+ ApplyBgworkerShared volatile *shared;
+} ApplyBgworkerState;
+
/* Main memory context for apply worker. Permanent during worker lifetime. */
extern PGDLLIMPORT MemoryContext ApplyContext;
+extern PGDLLIMPORT MemoryContext ApplyMessageContext;
+
+extern PGDLLIMPORT ApplyErrorCallbackArg apply_error_callback_arg;
+
+extern PGDLLIMPORT bool MySubscriptionValid;
+
+extern PGDLLIMPORT volatile ApplyBgworkerShared *MyParallelShared;
+
+extern PGDLLIMPORT List *subxactlist;
/* libpqreceiver connection */
extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn;
@@ -79,18 +145,22 @@ extern PGDLLIMPORT Subscription *MySubscription;
extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker;
extern PGDLLIMPORT bool in_remote_transaction;
+extern PGDLLIMPORT bool in_streamed_transaction;
+extern PGDLLIMPORT TransactionId stream_xid;
extern void logicalrep_worker_attach(int slot);
extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
bool only_running);
extern List *logicalrep_workers_find(Oid subid, bool only_running);
-extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
- Oid userid, Oid relid);
+extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
+ Oid userid, Oid relid,
+ dsm_handle subworker_dsm);
extern void logicalrep_worker_stop(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
extern int logicalrep_sync_worker_count(Oid subid);
+extern int logicalrep_apply_bgworker_count(Oid subid);
extern void ReplicationOriginNameForTablesync(Oid suboid, Oid relid,
char *originname, int szorgname);
@@ -103,10 +173,38 @@ extern void process_syncing_tables(XLogRecPtr current_lsn);
extern void invalidate_syncing_table_states(Datum arg, int cacheid,
uint32 hashvalue);
+extern void UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time,
+ bool reply);
+
+extern void apply_dispatch(StringInfo s);
+
+/* Function for apply error callback */
+extern void apply_error_callback(void *arg);
+
+extern void subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue);
+
+/* Apply background worker setup and interactions */
+extern ApplyBgworkerState *apply_bgworker_start(TransactionId xid);
+extern ApplyBgworkerState *apply_bgworker_find(TransactionId xid);
+extern void apply_bgworker_wait_for(ApplyBgworkerState *wstate,
+ ApplyBgworkerStatus wait_for_status);
+extern void apply_bgworker_send_data(ApplyBgworkerState *wstate, Size nbytes,
+ const void *data);
+extern void apply_bgworker_free(ApplyBgworkerState *wstate);
+extern void apply_bgworker_check_status(void);
+extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
+extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+
static inline bool
am_tablesync_worker(void)
{
return OidIsValid(MyLogicalRepWorker->relid);
}
+static inline bool
+am_apply_bgworker(void)
+{
+ return MyLogicalRepWorker->subworker;
+}
+
#endif /* WORKER_INTERNAL_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index c3ade01120..e35f199fd4 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -105,6 +105,7 @@ typedef enum
WAIT_EVENT_HASH_GROW_BUCKETS_ALLOCATE,
WAIT_EVENT_HASH_GROW_BUCKETS_ELECT,
WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE,
WAIT_EVENT_LOGICAL_SYNC_DATA,
WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE,
WAIT_EVENT_MQ_INTERNAL,
diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out
index ef0ebf96b9..4be537ea0b 100644
--- a/src/test/regress/expected/subscription.out
+++ b/src/test/regress/expected/subscription.out
@@ -219,7 +219,7 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
DROP SUBSCRIPTION regress_testsub;
-- fail - streaming must be boolean
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
-ERROR: streaming requires a Boolean value
+ERROR: streaming requires a Boolean value or "parallel"
-- now it works
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 34a76ceb60..4137dc77b4 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -116,6 +116,10 @@ Append
AppendPath
AppendRelInfo
AppendState
+ApplyBgworkerEntry
+ApplyBgworkerShared
+ApplyBgworkerState
+ApplyBgworkerStatus
ApplyErrorCallbackArg
ApplyExecutionData
ApplySubXactData
@@ -1485,6 +1489,7 @@ LogicalRepRelId
LogicalRepRelMapEntry
LogicalRepRelation
LogicalRepRollbackPreparedTxnData
+LogicalRepStreamAbortData
LogicalRepTupleData
LogicalRepTyp
LogicalRepWorker
--
2.23.0.windows.1
[application/octet-stream] v19-0002-Test-streaming-parallel-option-in-tap-test.patch (69.1K, ../../OS3PR01MB62752B29CEC2592E18236A9A9E909@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-v19-0002-Test-streaming-parallel-option-in-tap-test.patch)
download | inline diff:
From ef785a3bb9baed1a1f588e7905dac00050448b56 Mon Sep 17 00:00:00 2001
From: "shiy.fnst" <[email protected]>
Date: Fri, 13 May 2022 14:50:30 +0800
Subject: [PATCH v19 2/4] Test streaming parallel option in tap test
Change all TAP tests using the SUBSCRIPTION "streaming" parameter, so they
now test both 'on' and 'parallel' values.
---
src/test/subscription/t/015_stream.pl | 199 ++++---
src/test/subscription/t/016_stream_subxact.pl | 119 +++--
src/test/subscription/t/017_stream_ddl.pl | 188 ++++---
.../t/018_stream_subxact_abort.pl | 195 ++++---
.../t/019_stream_subxact_ddl_abort.pl | 110 +++-
.../subscription/t/022_twophase_cascade.pl | 363 +++++++------
.../subscription/t/023_twophase_stream.pl | 498 ++++++++++--------
7 files changed, 1035 insertions(+), 637 deletions(-)
diff --git a/src/test/subscription/t/015_stream.pl b/src/test/subscription/t/015_stream.pl
index 6561b189de..0bdd234935 100644
--- a/src/test/subscription/t/015_stream.pl
+++ b/src/test/subscription/t/015_stream.pl
@@ -8,6 +8,116 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Interleave a pair of transactions, each exceeding the 64kB limit.
+ my $in = '';
+ my $out = '';
+
+ my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
+
+ my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
+ on_error_stop => 0);
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ $in .= q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ };
+ $h->pump_nb;
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
+ DELETE FROM test_tab WHERE a > 5000;
+ COMMIT;
+ });
+
+ $in .= q{
+ COMMIT;
+ \q
+ };
+ $h->finish; # errors make the next test fail, so ignore them here
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'check extra columns contain local defaults');
+
+ # Test the streaming in binary mode
+ $node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain local defaults');
+
+ # Change the local values of the extra columns on the subscriber,
+ # update publisher, and check that subscriber retains the expected
+ # values. This is to ensure that non-streaming transactions behave
+ # properly after a streaming transaction.
+ $node_subscriber->safe_psql('postgres',
+ "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
+ );
+ $node_publisher->safe_psql('postgres',
+ "UPDATE test_tab SET b = md5(a::text)");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
+ );
+ is($result, qq(6667|6667|6667),
+ 'check extra columns contain locally changed data');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +147,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,82 +168,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Interleave a pair of transactions, each exceeding the 64kB limit.
-my $in = '';
-my $out = '';
-
-my $timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default);
-
-my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer,
- on_error_stop => 0);
-
-$in .= q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-};
-$h->pump_nb;
-
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 9999) s(i);
-DELETE FROM test_tab WHERE a > 5000;
-COMMIT;
-});
-
-$in .= q{
-COMMIT;
-\q
-};
-$h->finish; # errors make the next test fail, so ignore them here
-
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334), 'check extra columns contain local defaults');
-
-# Test the streaming in binary mode
-$node_subscriber->safe_psql('postgres',
- "ALTER SUBSCRIPTION tap_sub SET (binary = on)");
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(6667|6667|6667), 'check extra columns contain local defaults');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+);
-# Change the local values of the extra columns on the subscriber,
-# update publisher, and check that subscriber retains the expected
-# values. This is to ensure that non-streaming transactions behave
-# properly after a streaming transaction.
$node_subscriber->safe_psql('postgres',
- "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"
-);
-$node_publisher->safe_psql('postgres',
- "UPDATE test_tab SET b = md5(a::text)");
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel, binary = off)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"
-);
-is($result, qq(6667|6667|6667),
- 'check extra columns contain locally changed data');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/016_stream_subxact.pl b/src/test/subscription/t/016_stream_subxact.pl
index f27f1694f2..45429dddba 100644
--- a/src/test/subscription/t/016_stream_subxact.pl
+++ b/src/test/subscription/t/016_stream_subxact.pl
@@ -8,6 +8,72 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(1667|1667|1667),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +103,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,41 +124,26 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-# Insert, update and delete enough rows to exceed 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i);
-UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
-DELETE FROM test_tab WHERE mod(a,3) = 0;
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(1667|1667|1667),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/017_stream_ddl.pl b/src/test/subscription/t/017_stream_ddl.pl
index 0bce63b716..52dfef4780 100644
--- a/src/test/subscription/t/017_stream_ddl.pl
+++ b/src/test/subscription/t/017_stream_ddl.pl
@@ -8,6 +8,111 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (3, md5(3::text));
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (4, md5(4::text), -4);
+ COMMIT;
+ });
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ # a small (non-streamed) transaction with DDL and DML
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
+ is($result, qq(2002|1999|1002|1),
+ 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+ );
+
+ # A large (streamed) transaction with DDL and DML. One of the DDL is performed
+ # after DML to ensure that we invalidate the schema sent for test_tab so that
+ # the next transaction has to send the schema again.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
+ ALTER TABLE test_tab ADD COLUMN f INT;
+ COMMIT;
+ });
+
+ # A small transaction that won't get streamed. This is just to ensure that we
+ # send the schema again to reflect the last column added in the previous test.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab"
+ );
+ is($result, qq(5005|5002|4005|3004|5),
+ 'check data was copied to subscriber for both streaming and non-streaming transactions'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c, DROP COLUMN d, DROP COLUMN e, DROP COLUMN f;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +142,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,76 +163,25 @@ my $result =
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|0|0), 'check initial data was copied to subscriber');
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (3, md5(3::text));
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (4, md5(4::text), -4);
-COMMIT;
-});
-
-# large (streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i);
-COMMIT;
-});
-
-# a small (non-streamed) transaction with DDL and DML
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s1;
-INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e) FROM test_tab");
-is($result, qq(2002|1999|1002|1),
- 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# A large (streamed) transaction with DDL and DML. One of the DDL is performed
-# after DML to ensure that we invalidate the schema sent for test_tab so that
-# the next transaction has to send the schema again.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i);
-ALTER TABLE test_tab ADD COLUMN f INT;
-COMMIT;
-});
-
-# A small transaction that won't get streamed. This is just to ensure that we
-# send the schema again to reflect the last column added in the previous test.
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i);
-COMMIT;
-});
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$node_publisher->wait_for_catchup($appname);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab");
-is($result, qq(5005|5002|4005|3004|5),
- 'check data was copied to subscriber for both streaming and non-streaming transactions'
-);
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/018_stream_subxact_abort.pl b/src/test/subscription/t/018_stream_subxact_abort.pl
index 7155442e76..68f0e4b0d1 100644
--- a/src/test/subscription/t/018_stream_subxact_abort.pl
+++ b/src/test/subscription/t/018_stream_subxact_abort.pl
@@ -8,6 +8,113 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
+ ROLLBACK TO s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
+ SAVEPOINT s4;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
+ SAVEPOINT s5;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2000|0),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # large (streamed) transaction with subscriber receiving out of order
+ # subtransaction ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
+ RELEASE s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
+ ROLLBACK TO s1;
+ COMMIT;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0),
+ 'check rollback to savepoint was reflected on subscriber');
+
+ # large (streamed) transaction with subscriber receiving rollback
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
+ ROLLBACK;
+ });
+
+ $node_publisher->wait_for_catchup($appname);
+
+ $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE (a > 2)");
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -36,6 +143,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -53,81 +164,25 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i);
-ROLLBACK TO s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i);
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i);
-SAVEPOINT s4;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i);
-SAVEPOINT s5;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i);
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2000|0),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-# large (streamed) transaction with subscriber receiving out of order
-# subtransaction ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i);
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i);
-RELEASE s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i);
-ROLLBACK TO s1;
-COMMIT;
-});
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0),
- 'check rollback to savepoint was reflected on subscriber');
-
-# large (streamed) transaction with subscriber receiving rollback
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i);
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i);
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i);
-ROLLBACK;
-});
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(2500|0), 'check rollback was reflected on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
index dbd0fca4d1..b276063721 100644
--- a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
+++ b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl
@@ -9,6 +9,69 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # large (streamed) transaction with DDL, DML and ROLLBACKs
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
+ ALTER TABLE test_tab ADD COLUMN c INT;
+ SAVEPOINT s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
+ ALTER TABLE test_tab ADD COLUMN d INT;
+ SAVEPOINT s2;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
+ ALTER TABLE test_tab ADD COLUMN e INT;
+ SAVEPOINT s3;
+ INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
+ ALTER TABLE test_tab DROP COLUMN c;
+ ROLLBACK TO s1;
+ INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
+ COMMIT;
+ });
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ my $result =
+ $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c) FROM test_tab");
+ is($result, qq(1000|500),
+ 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+ );
+
+ # Cleanup the test data
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ DELETE FROM test_tab WHERE (a > 2);
+ ALTER TABLE test_tab DROP COLUMN c;
+ });
+ $node_publisher->wait_for_catchup($appname);
+}
+
# Create publisher node
my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
$node_publisher->init(allows_streaming => 'logical');
@@ -37,6 +100,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)"
);
@@ -54,35 +121,26 @@ my $result =
"SELECT count(*), count(c) FROM test_tab");
is($result, qq(2|0), 'check initial data was copied to subscriber');
-# large (streamed) transaction with DDL, DML and ROLLBACKs
-$node_publisher->safe_psql(
- 'postgres', q{
-BEGIN;
-INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i);
-ALTER TABLE test_tab ADD COLUMN c INT;
-SAVEPOINT s1;
-INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i);
-ALTER TABLE test_tab ADD COLUMN d INT;
-SAVEPOINT s2;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i);
-ALTER TABLE test_tab ADD COLUMN e INT;
-SAVEPOINT s3;
-INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i);
-ALTER TABLE test_tab DROP COLUMN c;
-ROLLBACK TO s1;
-INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i);
-COMMIT;
-});
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-$node_publisher->wait_for_catchup($appname);
-
-$result =
- $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c) FROM test_tab");
-is($result, qq(1000|500),
- 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
+
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
+
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
+
$node_subscriber->stop;
$node_publisher->stop;
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 7a797f37ba..0a4152d3be 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -11,6 +11,208 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming" parameter
+# so the same code can be run both for the streaming=on and streaming=parallel
+# cases.
+sub test_streaming
+{
+ my ($node_A, $node_B, $node_C, $appname_B, $appname_C, $streaming_mode) =
+ @_;
+
+ my $oldpid_B = $node_A->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';");
+ my $oldpid_C = $node_B->safe_psql(
+ 'postgres', "
+ SELECT pid FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';");
+
+ # Setup logical replication streaming mode
+
+ $node_B->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_B
+ SET (streaming = $streaming_mode);");
+ $node_C->safe_psql(
+ 'postgres', "
+ ALTER SUBSCRIPTION tap_sub_C
+ SET (streaming = $streaming_mode)");
+
+ # Wait for subscribers to finish initialization
+
+ $node_A->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_B FROM pg_stat_replication
+ WHERE application_name = '$appname_B' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+ $node_B->poll_query_until(
+ 'postgres', "
+ SELECT pid != $oldpid_C FROM pg_stat_replication
+ WHERE application_name = '$appname_C' AND state = 'streaming';"
+ ) or die "Timed out while waiting for apply to restart";
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber(s) after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" optparameterion is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_B->reload;
+
+ $node_C->append_conf('postgresql.conf', "log_min_messages = debug1");
+ $node_C->reload;
+ }
+
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ # Then 2PC PREPARE
+ $node_A->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_B->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_B->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_B->reload;
+
+ $node_C->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_C->append_conf('postgresql.conf', "log_min_messages = warning");
+ $node_C->reload;
+ }
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is prepared on subscriber(s)
+ my $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check that transaction was committed on subscriber(s)
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
+ );
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
+ );
+
+ # check the transaction state is ended on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber C');
+
+ ###############################
+ # Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
+ # 0. Cleanup from previous test leaving only 2 rows.
+ # 1. Insert one more row.
+ # 2. Record a SAVEPOINT.
+ # 3. Data is streamed using 2PC.
+ # 4. Do rollback to SAVEPOINT prior to the streamed inserts.
+ # 5. Then COMMIT PREPARED.
+ #
+ # Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (delete will be replicated)
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+
+ # 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
+ $node_A->safe_psql(
+ 'postgres', "
+ BEGIN;
+ INSERT INTO test_tab VALUES (9999, 'foobar');
+ SAVEPOINT sp_inner;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ ROLLBACK TO SAVEPOINT sp_inner;
+ PREPARE TRANSACTION 'outer';
+ ");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state prepared on subscriber(s)
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber C');
+
+ # 2PC COMMIT
+ $node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
+
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+
+ # check the transaction state is ended on subscriber
+ $result =
+ $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber B');
+ $result =
+ $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is ended on subscriber C');
+
+ # check inserts are visible at subscriber(s).
+ # All the streamed data (prior to the SAVEPOINT) should be rolled back.
+ # (9999, 'foobar') should be committed.
+ $result = $node_B->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber B');
+ $result =
+ $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber B');
+ $result = $node_C->safe_psql('postgres',
+ "SELECT count(*) FROM test_tab where b = 'foobar';");
+ is($result, qq(1), 'Rows committed are present on subscriber C');
+ $result =
+ $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
+ is($result, qq(3), 'Rows committed are present on subscriber C');
+
+ # Cleanup the test data
+ $node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
+ $node_A->wait_for_catchup($appname_B);
+ $node_B->wait_for_catchup($appname_C);
+}
+
###############################
# Setup a cascade of pub/sub nodes.
# node_A -> node_B -> node_C
@@ -260,160 +462,15 @@ is($result, qq(21), 'Rows committed are present on subscriber C');
# 2PC + STREAMING TESTS
# ---------------------
-my $oldpid_B = $node_A->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';");
-my $oldpid_C = $node_B->safe_psql(
- 'postgres', "
- SELECT pid FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';");
-
-# Setup logical replication (streaming = on)
-
-$node_B->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_B
- SET (streaming = on);");
-$node_C->safe_psql(
- 'postgres', "
- ALTER SUBSCRIPTION tap_sub_C
- SET (streaming = on)");
-
-# Wait for subscribers to finish initialization
-
-$node_A->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_B FROM pg_stat_replication
- WHERE application_name = '$appname_B' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-$node_B->poll_query_until(
- 'postgres', "
- SELECT pid != $oldpid_C FROM pg_stat_replication
- WHERE application_name = '$appname_C' AND state = 'streaming';"
-) or die "Timed out while waiting for apply to restart";
-
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber(s) after the commit.
-###############################
-
-# Insert, update and delete enough rows to exceed the 64kB limit.
-# Then 2PC PREPARE
-$node_A->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
+################################
+# Test using streaming mode 'on'
+################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'on');
-# check the transaction state is prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check that transaction was committed on subscriber(s)
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber B, and extra columns have local defaults'
-);
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber C, and extra columns have local defaults'
-);
-
-# check the transaction state is ended on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber C');
-
-###############################
-# Test 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT.
-# 0. Cleanup from previous test leaving only 2 rows.
-# 1. Insert one more row.
-# 2. Record a SAVEPOINT.
-# 3. Data is streamed using 2PC.
-# 4. Do rollback to SAVEPOINT prior to the streamed inserts.
-# 5. Then COMMIT PREPARED.
-#
-# Expect data after the SAVEPOINT is aborted leaving only 3 rows (= 2 original + 1 from step 1).
-###############################
-
-# First, delete the data except for 2 rows (delete will be replicated)
-$node_A->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# 2PC PREPARE with a nested ROLLBACK TO SAVEPOINT
-$node_A->safe_psql(
- 'postgres', "
- BEGIN;
- INSERT INTO test_tab VALUES (9999, 'foobar');
- SAVEPOINT sp_inner;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- ROLLBACK TO SAVEPOINT sp_inner;
- PREPARE TRANSACTION 'outer';
- ");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state prepared on subscriber(s)
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber C');
-
-# 2PC COMMIT
-$node_A->safe_psql('postgres', "COMMIT PREPARED 'outer';");
-
-$node_A->wait_for_catchup($appname_B);
-$node_B->wait_for_catchup($appname_C);
-
-# check the transaction state is ended on subscriber
-$result =
- $node_B->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber B');
-$result =
- $node_C->safe_psql('postgres', "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is ended on subscriber C');
-
-# check inserts are visible at subscriber(s).
-# All the streamed data (prior to the SAVEPOINT) should be rolled back.
-# (9999, 'foobar') should be committed.
-$result = $node_B->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber B');
-$result = $node_B->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber B');
-$result = $node_C->safe_psql('postgres',
- "SELECT count(*) FROM test_tab where b = 'foobar';");
-is($result, qq(1), 'Rows committed are present on subscriber C');
-$result = $node_C->safe_psql('postgres', "SELECT count(*) FROM test_tab;");
-is($result, qq(3), 'Rows committed are present on subscriber C');
+######################################
+# Test using streaming mode 'parallel'
+######################################
+test_streaming($node_A, $node_B, $node_C, $appname_B, $appname_C, 'parallel');
###############################
# check all the cleanup
diff --git a/src/test/subscription/t/023_twophase_stream.pl b/src/test/subscription/t/023_twophase_stream.pl
index d8475d25a4..b89414ab74 100644
--- a/src/test/subscription/t/023_twophase_stream.pl
+++ b/src/test/subscription/t/023_twophase_stream.pl
@@ -8,6 +8,266 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
+# Encapsulate all the common test steps which are related to "streaming"
+# parameter so the same code can be run both for the streaming=on and
+# streaming=parallel cases.
+sub test_streaming
+{
+ my ($node_publisher, $node_subscriber, $appname, $is_parallel) = @_;
+
+ ###############################
+ # Test 2PC PREPARE / COMMIT PREPARED.
+ # 1. Data is streamed as a 2PC transaction.
+ # 2. Then do commit prepared.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ ###############################
+
+ # Check that a background worker starts if "streaming" parameter is
+ # specified as "parallel". We have to look for the DEBUG1 log messages
+ # about that, so temporarily bump up the log verbosity.
+ if ($is_parallel)
+ {
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = debug1");
+ $node_subscriber->reload;
+ }
+
+ # check that 2PC gets replicated to subscriber
+ # Insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ if ($is_parallel)
+ {
+ $node_subscriber->wait_for_log(qr/\[Apply BGW #\d+\] started/, 0);
+ $node_subscriber->append_conf('postgresql.conf',
+ "log_min_messages = warning");
+ $node_subscriber->reload;
+ }
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ my $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ ###############################
+ # Test 2PC PREPARE / ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. Do rollback prepared.
+ #
+ # Expect data rolls back leaving only the original 2 rows.
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(2|2|2),
+ 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
+ # 1. insert, update and delete enough rows to exceed the 64kB limit.
+ # 2. Then server crashes before the 2PC transaction is committed.
+ # 3. After servers are restarted the pending transaction is committed.
+ #
+ # Expect all data is replicated on subscriber side after the commit.
+ # Note: both publisher and subscriber do crash/restart.
+ ###############################
+
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_subscriber->stop('immediate');
+ $node_publisher->stop('immediate');
+
+ $node_publisher->start;
+ $node_subscriber->start;
+
+ # commit post the restart
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+ $node_publisher->wait_for_catchup($appname);
+
+ # check inserts are visible
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3334|3334|3334),
+ 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ ###############################
+ # Do INSERT after the PREPARE but before ROLLBACK PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a ROLLBACK PREPARED.
+ #
+ # Expect the 2PC data rolls back leaving only 3 rows on the subscriber
+ # (the original 2 + inserted 1).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets aborted
+ $node_publisher->safe_psql('postgres',
+ "ROLLBACK PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is aborted on subscriber,
+ # but the extra INSERT outside of the 2PC still was replicated
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3|3|3),
+ 'check the outside insert was copied to subscriber');
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is aborted on subscriber');
+
+ ###############################
+ # Do INSERT after the PREPARE but before COMMIT PREPARED.
+ # 1. Table is deleted back to 2 rows which are replicated on subscriber.
+ # 2. Data is streamed using 2PC.
+ # 3. A single row INSERT is done which is after the PREPARE.
+ # 4. Then do a COMMIT PREPARED.
+ #
+ # Expect 2PC data + the extra row are on the subscriber
+ # (the 3334 + inserted 1 = 3335).
+ ###############################
+
+ # First, delete the data except for 2 rows (will be replicated)
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+
+ # Then insert, update and delete enough rows to exceed the 64kB limit.
+ $node_publisher->safe_psql(
+ 'postgres', q{
+ BEGIN;
+ INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
+ UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
+ DELETE FROM test_tab WHERE mod(a,3) = 0;
+ PREPARE TRANSACTION 'test_prepared_tab';});
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is in prepared state on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(1), 'transaction is prepared on subscriber');
+
+ # Insert a different record (now we are outside of the 2PC transaction)
+ # Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
+ $node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab VALUES (99999, 'foobar')");
+
+ # 2PC transaction gets committed
+ $node_publisher->safe_psql('postgres',
+ "COMMIT PREPARED 'test_prepared_tab';");
+
+ $node_publisher->wait_for_catchup($appname);
+
+ # check that transaction is committed on subscriber
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*), count(c), count(d = 999) FROM test_tab");
+ is($result, qq(3335|3335|3335),
+ 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
+ );
+
+ $result = $node_subscriber->safe_psql('postgres',
+ "SELECT count(*) FROM pg_prepared_xacts;");
+ is($result, qq(0), 'transaction is committed on subscriber');
+
+ # Cleanup the test data
+ $node_publisher->safe_psql('postgres',
+ "DELETE FROM test_tab WHERE a > 2;");
+ $node_publisher->wait_for_catchup($appname);
+}
+
###############################
# Setup
###############################
@@ -48,6 +308,10 @@ $node_publisher->safe_psql('postgres',
"CREATE PUBLICATION tap_pub FOR TABLE test_tab");
my $appname = 'tap_sub';
+
+################################
+# Test using streaming mode 'on'
+################################
$node_subscriber->safe_psql(
'postgres', "
CREATE SUBSCRIPTION tap_sub
@@ -70,236 +334,30 @@ my $twophase_query =
$node_subscriber->poll_query_until('postgres', $twophase_query)
or die "Timed out while waiting for subscriber to enable twophase";
-###############################
# Check initial data was copied to subscriber
-###############################
my $result = $node_subscriber->safe_psql('postgres',
"SELECT count(*), count(c), count(d = 999) FROM test_tab");
is($result, qq(2|2|2), 'check initial data was copied to subscriber');
-###############################
-# Test 2PC PREPARE / COMMIT PREPARED.
-# 1. Data is streamed as a 2PC transaction.
-# 2. Then do commit prepared.
-#
-# Expect all data is replicated on subscriber side after the commit.
-###############################
-
-# check that 2PC gets replicated to subscriber
-# Insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
-);
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
-
-###############################
-# Test 2PC PREPARE / ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. Do rollback prepared.
-#
-# Expect data rolls back leaving only the original 2 rows.
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(2|2|2),
- 'Rows inserted by 2PC are rolled back, leaving only the original 2 rows');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Check that 2PC COMMIT PREPARED is decoded properly on crash restart.
-# 1. insert, update and delete enough rows to exceed the 64kB limit.
-# 2. Then server crashes before the 2PC transaction is committed.
-# 3. After servers are restarted the pending transaction is committed.
-#
-# Expect all data is replicated on subscriber side after the commit.
-# Note: both publisher and subscriber do crash/restart.
-###############################
-
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_subscriber->stop('immediate');
-$node_publisher->stop('immediate');
-
-$node_publisher->start;
-$node_subscriber->start;
-
-# commit post the restart
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-$node_publisher->wait_for_catchup($appname);
+test_streaming($node_publisher, $node_subscriber, $appname, 0);
-# check inserts are visible
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3334|3334|3334),
- 'Rows inserted by 2PC have committed on subscriber, and extra columns contain local defaults'
+######################################
+# Test using streaming mode 'parallel'
+######################################
+my $oldpid = $node_publisher->safe_psql('postgres',
+ "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
);
-###############################
-# Do INSERT after the PREPARE but before ROLLBACK PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a ROLLBACK PREPARED.
-#
-# Expect the 2PC data rolls back leaving only 3 rows on the subscriber
-# (the original 2 + inserted 1).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separate primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets aborted
-$node_publisher->safe_psql('postgres',
- "ROLLBACK PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is aborted on subscriber,
-# but the extra INSERT outside of the 2PC still was replicated
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3|3|3), 'check the outside insert was copied to subscriber');
-
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is aborted on subscriber');
-
-###############################
-# Do INSERT after the PREPARE but before COMMIT PREPARED.
-# 1. Table is deleted back to 2 rows which are replicated on subscriber.
-# 2. Data is streamed using 2PC.
-# 3. A single row INSERT is done which is after the PREPARE.
-# 4. Then do a COMMIT PREPARED.
-#
-# Expect 2PC data + the extra row are on the subscriber
-# (the 3334 + inserted 1 = 3335).
-###############################
-
-# First, delete the data except for 2 rows (will be replicated)
-$node_publisher->safe_psql('postgres', "DELETE FROM test_tab WHERE a > 2;");
-
-# Then insert, update and delete enough rows to exceed the 64kB limit.
-$node_publisher->safe_psql(
- 'postgres', q{
- BEGIN;
- INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i);
- UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0;
- DELETE FROM test_tab WHERE mod(a,3) = 0;
- PREPARE TRANSACTION 'test_prepared_tab';});
-
-$node_publisher->wait_for_catchup($appname);
-
-# check that transaction is in prepared state on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(1), 'transaction is prepared on subscriber');
-
-# Insert a different record (now we are outside of the 2PC transaction)
-# Note: the 2PC transaction still holds row locks so make sure this insert is for a separare primary key
-$node_publisher->safe_psql('postgres',
- "INSERT INTO test_tab VALUES (99999, 'foobar')");
-
-# 2PC transaction gets committed
-$node_publisher->safe_psql('postgres',
- "COMMIT PREPARED 'test_prepared_tab';");
-
-$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql('postgres',
+ "ALTER SUBSCRIPTION tap_sub SET(streaming = parallel)");
-# check that transaction is committed on subscriber
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*), count(c), count(d = 999) FROM test_tab");
-is($result, qq(3335|3335|3335),
- 'Rows inserted by 2PC (as well as outside insert) have committed on subscriber, and extra columns contain local defaults'
-);
+$node_publisher->poll_query_until('postgres',
+ "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname' AND state = 'streaming';"
+ )
+ or die
+ "Timed out while waiting for apply to restart after changing SUBSCRIPTION";
-$result = $node_subscriber->safe_psql('postgres',
- "SELECT count(*) FROM pg_prepared_xacts;");
-is($result, qq(0), 'transaction is committed on subscriber');
+test_streaming($node_publisher, $node_subscriber, $appname, 1);
###############################
# check all the cleanup
--
2.23.0.windows.1
[application/octet-stream] v19-0003-Add-some-checks-before-using-apply-background-wo.patch (37.8K, ../../OS3PR01MB62752B29CEC2592E18236A9A9E909@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-v19-0003-Add-some-checks-before-using-apply-background-wo.patch)
download | inline diff:
From 1cb19ad525fc6903f9b55568c16254dca3c6736c Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 11:23:52 +0800
Subject: [PATCH v19 3/4] Add some checks before using apply background worker
to apply changes.
streaming=parallel mode has two requirements:
1) The unique column in the relation on the subscriber-side should also be the
unique column on the publisher-side;
2) There cannot be any non-immutable functions used by the subscriber-side
replicated table. Look for functions in the following places:
* a. Trigger functions
* b. Column default value expressions and domain constraints
* c. Constraint expressions
* d. Foreign keys
---
doc/src/sgml/ref/create_subscription.sgml | 4 +
.../replication/logical/applybgworker.c | 44 ++
src/backend/replication/logical/proto.c | 88 +++-
src/backend/replication/logical/relation.c | 201 +++++++++
src/backend/replication/logical/tablesync.c | 1 +
src/backend/replication/logical/worker.c | 23 +-
src/backend/utils/cache/typcache.c | 17 +
src/include/replication/logicalproto.h | 1 +
src/include/replication/logicalrelation.h | 15 +
src/include/replication/worker_internal.h | 1 +
src/include/utils/typcache.h | 2 +
.../subscription/t/022_twophase_cascade.pl | 6 +
.../subscription/t/032_streaming_apply.pl | 380 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
14 files changed, 775 insertions(+), 9 deletions(-)
create mode 100644 src/test/subscription/t/032_streaming_apply.pl
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index b08e4b5580..832899570f 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -240,6 +240,10 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
transaction is committed. Note that if an error happens when
applying changes in a background worker, the finish LSN of the
remote transaction might not be reported in the server log.
+ <literal>parallel</literal> mode has two requirements: 1) the unique
+ column in the relation on the subscriber-side should also be the
+ unique column on the publisher-side; 2) there cannot be any
+ non-immutable functions used by the subscriber-side replicated table.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index aa222490a0..89c712f785 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -800,3 +800,47 @@ apply_bgworker_subxact_info_add(TransactionId current_xid)
MemoryContextSwitchTo(oldctx);
}
}
+
+/*
+ * Check if changes on this relation can be applied by an apply background
+ * worker.
+ *
+ * Although the commit order is maintained only allowing one process to commit
+ * at a time, the access order to the relation has changed. This could cause
+ * unexpected problems if the unique column on the replicated table is
+ * inconsistent with the publisher-side or contains non-immutable functions
+ * when applying transactions in the apply background worker.
+ */
+void
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
+{
+ /* Skip check if not an apply background worker. */
+ if (!am_apply_bgworker())
+ return;
+
+ /*
+ * Partition table checks are done later in function
+ * apply_handle_tuple_routing.
+ */
+ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ return;
+
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel_apply == PARALLEL_APPLY_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot replicate target relation \"%s.%s\" using "
+ "subscription parameter streaming=parallel",
+ rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change to use subscription parameter "
+ "streaming=on.")));
+}
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index 47bd811fb7..511bd9c052 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -23,7 +23,8 @@
/*
* Protocol message flags.
*/
-#define LOGICALREP_IS_REPLICA_IDENTITY 1
+#define ATTR_IS_REPLICA_IDENTITY (1 << 0)
+#define ATTR_IS_UNIQUE (1 << 1)
#define MESSAGE_TRANSACTIONAL (1<<0)
#define TRUNCATE_CASCADE (1<<0)
@@ -40,6 +41,68 @@ static void logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple);
static void logicalrep_write_namespace(StringInfo out, Oid nspid);
static const char *logicalrep_read_namespace(StringInfo in);
+static Bitmapset *RelationGetUniqueKeyBitmap(Relation rel);
+
+/*
+ * RelationGetUniqueKeyBitmap -- get a bitmap of unique attribute numbers
+ *
+ * This is similar to RelationGetIdentityKeyBitmap(), but returns a bitmap of
+ * index attribute numbers for all unique indexes.
+ */
+static Bitmapset *
+RelationGetUniqueKeyBitmap(Relation rel)
+{
+ List *indexoidlist = NIL;
+ ListCell *indexoidscan;
+ Bitmapset *attunique = NULL;
+
+ if (!rel->rd_rel->relhasindex)
+ return NULL;
+
+ indexoidlist = RelationGetIndexList(rel);
+
+ foreach(indexoidscan, indexoidlist)
+ {
+ Oid indexoid = lfirst_oid(indexoidscan);
+ Relation indexRel;
+ int i;
+
+ /* Look up the description for index */
+ indexRel = RelationIdGetRelation(indexoid);
+
+ if (!RelationIsValid(indexRel))
+ elog(ERROR, "could not open relation with OID %u", indexoid);
+
+ if (!indexRel->rd_index->indisunique)
+ {
+ RelationClose(indexRel);
+ continue;
+ }
+
+ /* Add referenced attributes to idindexattrs */
+ for (i = 0; i < indexRel->rd_index->indnatts; i++)
+ {
+ int attrnum = indexRel->rd_index->indkey.values[i];
+
+ /*
+ * We don't include non-key columns into idindexattrs
+ * bitmaps. See RelationGetIndexAttrBitmap.
+ */
+ if (attrnum != 0)
+ {
+ if (i < indexRel->rd_index->indnkeyatts &&
+ !bms_is_member(attrnum - FirstLowInvalidHeapAttributeNumber, attunique))
+ attunique = bms_add_member(attunique,
+ attrnum - FirstLowInvalidHeapAttributeNumber);
+ }
+ }
+ RelationClose(indexRel);
+ }
+ list_free(indexoidlist);
+
+ return attunique;
+}
+
/*
* Check if a column is covered by a column list.
*
@@ -933,7 +996,8 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
TupleDesc desc;
int i;
uint16 nliveatts = 0;
- Bitmapset *idattrs = NULL;
+ Bitmapset *idattrs = NULL,
+ *attunique = NULL;
bool replidentfull;
desc = RelationGetDescr(rel);
@@ -958,6 +1022,9 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (!replidentfull)
idattrs = RelationGetIdentityKeyBitmap(rel);
+ /* fetch bitmap of UNIQUE attributes */
+ attunique = RelationGetUniqueKeyBitmap(rel);
+
/* send the attributes */
for (i = 0; i < desc->natts; i++)
{
@@ -974,7 +1041,11 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
if (replidentfull ||
bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
idattrs))
- flags |= LOGICALREP_IS_REPLICA_IDENTITY;
+ flags |= ATTR_IS_REPLICA_IDENTITY;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ attunique))
+ flags |= ATTR_IS_UNIQUE;
pq_sendbyte(out, flags);
@@ -989,6 +1060,7 @@ logicalrep_write_attrs(StringInfo out, Relation rel, Bitmapset *columns)
}
bms_free(idattrs);
+ bms_free(attunique);
}
/*
@@ -1001,7 +1073,8 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
int natts;
char **attnames;
Oid *atttyps;
- Bitmapset *attkeys = NULL;
+ Bitmapset *attkeys = NULL,
+ *attunique = NULL;
natts = pq_getmsgint(in, 2);
attnames = palloc(natts * sizeof(char *));
@@ -1014,9 +1087,13 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
/* Check for replica identity column */
flags = pq_getmsgbyte(in);
- if (flags & LOGICALREP_IS_REPLICA_IDENTITY)
+ if (flags & ATTR_IS_REPLICA_IDENTITY)
attkeys = bms_add_member(attkeys, i);
+ /* Check for unique column */
+ if (flags & ATTR_IS_UNIQUE)
+ attunique = bms_add_member(attunique, i);
+
/* attribute name */
attnames[i] = pstrdup(pq_getmsgstring(in));
@@ -1030,6 +1107,7 @@ logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel)
rel->attnames = attnames;
rel->atttyps = atttyps;
rel->attkeys = attkeys;
+ rel->attunique = attunique;
rel->natts = natts;
}
diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c
index e989047681..37e410b8d0 100644
--- a/src/backend/replication/logical/relation.c
+++ b/src/backend/replication/logical/relation.c
@@ -19,12 +19,19 @@
#include "access/table.h"
#include "catalog/namespace.h"
+#include "catalog/pg_proc.h"
#include "catalog/pg_subscription_rel.h"
+#include "commands/trigger.h"
#include "executor/executor.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "replication/logicalrelation.h"
#include "replication/worker_internal.h"
+#include "rewrite/rewriteHandler.h"
#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
static MemoryContext LogicalRepRelMapContext = NULL;
@@ -91,6 +98,26 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
}
}
+/*
+ * Relcache invalidation callback to reset parallel flag.
+ */
+static void
+logicalrep_relmap_reset_parallel_cb(Datum arg, int cacheid, uint32 hashvalue)
+{
+ HASH_SEQ_STATUS hash_seq;
+ LogicalRepRelMapEntry *entry;
+
+ if (LogicalRepRelMap == NULL)
+ return;
+
+ hash_seq_init(&hash_seq, LogicalRepRelMap);
+ while ((entry = hash_seq_search(&hash_seq)) != NULL)
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNKNOWN;
+ entry->localrelvalid = false;
+ }
+}
+
/*
* Initialize the relation map cache.
*/
@@ -116,6 +143,9 @@ logicalrep_relmap_init(void)
/* Watch for invalidation events. */
CacheRegisterRelcacheCallback(logicalrep_relmap_invalidate_cb,
(Datum) 0);
+ CacheRegisterSyscacheCallback(PROCOID,
+ logicalrep_relmap_reset_parallel_cb,
+ (Datum) 0);
}
/*
@@ -142,6 +172,7 @@ logicalrep_relmap_free_entry(LogicalRepRelMapEntry *entry)
pfree(remoterel->atttyps);
}
bms_free(remoterel->attkeys);
+ bms_free(remoterel->attunique);
if (entry->attrmap)
free_attrmap(entry->attrmap);
@@ -190,6 +221,7 @@ logicalrep_relmap_update(LogicalRepRelation *remoterel)
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
MemoryContextSwitchTo(oldctx);
}
@@ -310,6 +342,168 @@ logicalrep_rel_mark_updatable(LogicalRepRelMapEntry *entry)
}
}
+/*
+ * Check if changes on one relation can be applied by an apply background
+ * worker and assign the 'parallel_apply' flag.
+ *
+ * There are two requirements for applying changes in an apply background
+ * worker: 1) The unique column in the relation on the subscriber-side should
+ * also be the unique column on the publisher-side; 2) There cannot be any
+ * non-immutable functions used by the subscriber-side.
+ *
+ * We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
+ * on one relation can not be applied by an apply background worker and leave
+ * it to apply_bgworker_relation_check() to throw the actual error if needed.
+ */
+static void
+logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
+{
+ Bitmapset *ukey;
+ int i;
+ TupleDesc tupdesc;
+ int attnum;
+ List *fkeys = NIL;
+
+ /* Fast path if 'parallel_apply' flag is already known. */
+ if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
+ return;
+
+ /* Initialize the flag. */
+ entry->parallel_apply = PARALLEL_APPLY_SAFE;
+
+ /*
+ * First, check if the unique column in the relation on the subscriber-side
+ * is also the unique column on the publisher-side.
+ */
+ ukey = RelationGetIndexAttrBitmap(entry->localrel,
+ INDEX_ATTR_BITMAP_KEY);
+
+ if (ukey)
+ {
+ i = -1;
+ while ((i = bms_next_member(ukey, i)) >= 0)
+ {
+ attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
+
+ if (entry->attrmap->attnums[attnum] < 0 ||
+ !bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+
+ bms_free(ukey);
+ }
+
+ /*
+ * Then, check if there is any non-immutable function used by the local
+ * table. Look for functions in the following places:
+ * a. trigger functions;
+ * b. Column default value expressions and domain constraints;
+ * c. Constraint expressions;
+ * d. Foreign keys.
+ */
+ /* Check the trigger functions. */
+ if (entry->localrel->trigdesc != NULL)
+ {
+ for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
+ {
+ Trigger *trig = entry->localrel->trigdesc->triggers + i;
+
+ if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
+ trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
+ continue;
+
+ if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the columns. */
+ tupdesc = RelationGetDescr(entry->localrel);
+ for (attnum = 0; attnum < tupdesc->natts; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
+
+ /* We don't need info for dropped or generated attributes */
+ if (att->attisdropped || att->attgenerated)
+ continue;
+
+ /*
+ * We don't need to check columns that only exist on the
+ * subscriber
+ */
+ if (entry->attrmap->attnums[attnum] < 0)
+ continue;
+
+ if (att->atthasdef)
+ {
+ Node *defaultexpr;
+
+ defaultexpr = build_column_default(entry->localrel, attnum + 1);
+ if (contain_mutable_functions(defaultexpr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+
+ /*
+ * If the column is of a DOMAIN type, determine whether
+ * that domain has any CHECK expressions that are not
+ * immutable.
+ */
+ if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
+ {
+ List *domain_constraints;
+ ListCell *lc;
+
+ domain_constraints = GetDomainConstraints(att->atttypid);
+
+ foreach(lc, domain_constraints)
+ {
+ DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
+
+ if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+ }
+
+ /* Check the constraints. */
+ if (tupdesc->constr)
+ {
+ ConstrCheck *check = tupdesc->constr->check;
+
+ /*
+ * Determine if there are any CHECK constraints which
+ * contains non-immutable function.
+ */
+ for (i = 0; i < tupdesc->constr->num_check; i++)
+ {
+ Expr *check_expr = stringToNode(check[i].ccbin);
+
+ if (contain_mutable_functions((Node *) check_expr))
+ {
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+ return;
+ }
+ }
+ }
+
+ /* Check the foreign keys. */
+ fkeys = RelationGetFKeyList(entry->localrel);
+ if (fkeys)
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
+}
+
/*
* Open the local relation associated with the remote one.
*
@@ -438,6 +632,9 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
*/
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel_apply(entry);
+
entry->localrelvalid = true;
}
@@ -653,6 +850,7 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
}
entry->remoterel.replident = remoterel->replident;
entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
+ entry->remoterel.attunique = bms_copy(remoterel->attunique);
}
entry->localrel = partrel;
@@ -696,6 +894,9 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root,
/* Set if the table's replica identity is enough to apply update/delete. */
logicalrep_rel_mark_updatable(entry);
+ /* Set if changes could be applied in the apply background worker. */
+ logicalrep_rel_mark_parallel_apply(entry);
+
entry->localrelvalid = true;
/* state and statelsn are left set to 0. */
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8ffba7e2e5..3cdbf8b457 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -884,6 +884,7 @@ fetch_remote_table_info(char *nspname, char *relname,
lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
lrel->attkeys = NULL;
+ lrel->attunique = NULL;
/*
* Store the columns as a list of names. Ignore those that are not
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 2aa7797628..41793d26c2 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -1391,6 +1391,14 @@ apply_handle_stream_stop(StringInfo s)
{
char action = LOGICAL_REP_MSG_STREAM_STOP;
+ /*
+ * Unlike stream_commit, we don't need to wait here for stream_stop to
+ * finish. Allowing the other transaction to be applied before
+ * stream_stop is finished can lead to failures if the unique
+ * index/constraint is different between publisher and subscriber. But
+ * for such cases, we don't allow streamed transactions to be applied
+ * in parallel. See apply_bgworker_relation_check.
+ */
apply_bgworker_send_data(stream_apply_worker, 1, &action);
elog(DEBUG1, "stopped streaming of xid %u, %u changes streamed", stream_xid, nchanges);
@@ -2044,6 +2052,8 @@ apply_handle_insert(StringInfo s)
/* Set relation for error callback */
apply_error_callback_arg.rel = rel;
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2187,6 +2197,8 @@ apply_handle_update(StringInfo s)
/* Check if we can do the update. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2355,6 +2367,8 @@ apply_handle_delete(StringInfo s)
/* Check if we can do the delete. */
check_relation_updatable(rel);
+ apply_bgworker_relation_check(rel);
+
/* Initialize the executor state. */
edata = create_edata_for_relation(rel);
estate = edata->estate;
@@ -2540,13 +2554,14 @@ apply_handle_tuple_routing(ApplyExecutionData *edata,
}
MemoryContextSwitchTo(oldctx);
+ part_entry = logicalrep_partition_open(relmapentry, partrel,
+ attrmap);
+
/* Check if we can do the update or delete on the leaf partition. */
if (operation == CMD_UPDATE || operation == CMD_DELETE)
- {
- part_entry = logicalrep_partition_open(relmapentry, partrel,
- attrmap);
check_relation_updatable(part_entry);
- }
+
+ apply_bgworker_relation_check(part_entry);
switch (operation)
{
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 808f9ebd0d..b248899d82 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -2540,6 +2540,23 @@ compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2)
return 0;
}
+/*
+ * GetDomainConstraints --- get DomainConstraintState list of specified domain type
+ */
+List *
+GetDomainConstraints(Oid type_id)
+{
+ TypeCacheEntry *typentry;
+ List *constraints = NIL;
+
+ typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
+
+ if(typentry->domainData != NULL)
+ constraints = typentry->domainData->constraints;
+
+ return constraints;
+}
+
/*
* Load (or re-load) the enumData member of the typcache entry.
*/
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index eb0fd24fd8..4395c11f75 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -113,6 +113,7 @@ typedef struct LogicalRepRelation
char replident; /* replica identity */
char relkind; /* remote relation kind */
Bitmapset *attkeys; /* Bitmap of key columns */
+ Bitmapset *attunique; /* Bitmap of unique columns */
} LogicalRepRelation;
/* Type mapping info */
diff --git a/src/include/replication/logicalrelation.h b/src/include/replication/logicalrelation.h
index 78cd7e77f5..8011e648d7 100644
--- a/src/include/replication/logicalrelation.h
+++ b/src/include/replication/logicalrelation.h
@@ -15,6 +15,19 @@
#include "access/attmap.h"
#include "replication/logicalproto.h"
+/*
+ * States to determine if changes on one relation can be applied using an
+ * apply background worker.
+ */
+typedef enum ParalleApplySafety
+{
+ PARALLEL_APPLY_UNKNOWN = 0, /* unknown */
+ PARALLEL_APPLY_SAFE, /* Can apply changes in an apply background
+ worker */
+ PARALLEL_APPLY_UNSAFE /* Can not apply changes in an apply background
+ worker */
+} ParalleApplySafety;
+
typedef struct LogicalRepRelMapEntry
{
LogicalRepRelation remoterel; /* key is remoterel.remoteid */
@@ -31,6 +44,8 @@ typedef struct LogicalRepRelMapEntry
Relation localrel; /* relcache entry (NULL when closed) */
AttrMap *attrmap; /* map of local attributes to remote ones */
bool updatable; /* Can apply updates/deletes? */
+ ParalleApplySafety parallel_apply; /* Can apply changes in an apply
+ background worker? */
/* Sync state. */
char state;
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index a3560d4904..1c0db05c8a 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -194,6 +194,7 @@ extern void apply_bgworker_free(ApplyBgworkerState *wstate);
extern void apply_bgworker_check_status(void);
extern void apply_bgworker_set_status(ApplyBgworkerStatus status);
extern void apply_bgworker_subxact_info_add(TransactionId current_xid);
+extern void apply_bgworker_relation_check(LogicalRepRelMapEntry *rel);
static inline bool
am_tablesync_worker(void)
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index 431ad7f1b3..ed7c2e7f48 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -199,6 +199,8 @@ extern uint64 assign_record_type_identifier(Oid type_id, int32 typmod);
extern int compare_values_of_enum(TypeCacheEntry *tcache, Oid arg1, Oid arg2);
+extern List *GetDomainConstraints(Oid type_id);
+
extern size_t SharedRecordTypmodRegistryEstimate(void);
extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
diff --git a/src/test/subscription/t/022_twophase_cascade.pl b/src/test/subscription/t/022_twophase_cascade.pl
index 0a4152d3be..30a01f7305 100644
--- a/src/test/subscription/t/022_twophase_cascade.pl
+++ b/src/test/subscription/t/022_twophase_cascade.pl
@@ -39,6 +39,12 @@ sub test_streaming
ALTER SUBSCRIPTION tap_sub_C
SET (streaming = $streaming_mode)");
+ if ($streaming_mode eq 'parallel')
+ {
+ $node_C->safe_psql(
+ 'postgres', "ALTER TABLE test_tab ALTER c DROP DEFAULT");
+ }
+
# Wait for subscribers to finish initialization
$node_A->poll_query_until(
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
new file mode 100644
index 0000000000..7f8bfa6745
--- /dev/null
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -0,0 +1,380 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test the restrictions of streaming mode "parallel" in logical replication
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $offset = 0;
+
+# Create publisher node
+my $node_publisher = PostgreSQL::Test::Cluster->new('publisher');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->append_conf('postgresql.conf',
+ 'logical_decoding_work_mem = 64kB');
+$node_publisher->start;
+
+# Create subscriber node
+my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber');
+$node_subscriber->init;
+$node_subscriber->start;
+
+# Setup structure on publisher
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar)");
+
+# Setup structure on subscriber
+# We need to test normal table and partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab (a int primary key, b varchar)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partitioned (a int primary key, b varchar) PARTITION BY RANGE(a)");
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE test_tab_partition (LIKE test_tab_partitioned)");
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partitioned ATTACH PARTITION test_tab_partition DEFAULT"
+);
+
+# Setup logical replication
+my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub FOR TABLE test_tab");
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION tap_pub_partitioned FOR TABLE test_tab_partitioned");
+
+my $appname = 'tap_sub';
+$node_subscriber->safe_psql(
+ 'postgres', "
+ CREATE SUBSCRIPTION tap_sub
+ CONNECTION '$publisher_connstr application_name=$appname'
+ PUBLICATION tap_pub, tap_pub_partitioned
+ WITH (streaming = parallel, copy_data = false)");
+
+$node_publisher->wait_for_catchup($appname);
+
+# It is not allowed that the unique index on the publisher and the subscriber
+# is different. Check the error reported by background worker in this case.
+# First we check the unique index on normal table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+my $result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Then we check the unique index on partition table.
+$node_subscriber->safe_psql('postgres',
+ "CREATE UNIQUE INDEX test_tab_b_partition_idx ON test_tab_partition (b)");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the unique index on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000), 'data replicated to subscriber after dropping index');
+
+# Triggers which execute non-immutable function are not allowed on the
+# subscriber side. Check the error reported by background worker in this case.
+# First we check the trigger function on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE FUNCTION trigger_func() RETURNS TRIGGER AS \$\$
+ BEGIN
+ RETURN NULL;
+ END
+\$\$ language plpgsql;
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# Then we check the trigger function on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TRIGGER insert_trig
+BEFORE INSERT ON test_tab_partition
+FOR EACH ROW EXECUTE PROCEDURE trigger_func();
+ALTER TABLE test_tab_partition ENABLE REPLICA TRIGGER insert_trig;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the trigger on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+
+# It is not allowed that column default value expression contains a
+# non-immutable function on the subscriber side. Check the error reported by
+# background worker in this case.
+# First we check the column default value expression on normal table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# Then we check the column default value expression on partition table.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b SET DEFAULT random()");
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop default value on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping default value expression');
+
+# It is not allowed that domain constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case.
+# Because the column type of the partition table must be the same as its parent
+# table, only test normal table here.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE DOMAIN test_domain AS int CHECK(VALUE > random());
+ALTER TABLE test_tab ALTER COLUMN a TYPE test_domain;
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop domain constraint expression on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping domain constraint expression'
+);
+
+# It is not allowed that constraint expression contains a non-immutable function
+# on the subscriber side. Check the error reported by background worker in this
+# case.
+# First we check the constraint expression on normal table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# Then we check the constraint expression on partition table.
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_con check (a > random());
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab_partitioned");
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(0),
+ 'data replicated to subscriber after dropping constraint expression');
+
+# It is not allowed that foreign key on the subscriber side. Check the error
+# reported by background worker in this case.
+# First we check the foreign key on normal table.
+$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_f (a int primary key);
+ALTER TABLE test_tab ADD CONSTRAINT test_tabfk FOREIGN KEY(a) REFERENCES test_tab_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+# Then we check the foreign key on partition table.
+$node_publisher->wait_for_catchup($appname);
+$node_subscriber->safe_psql(
+ 'postgres', qq{
+CREATE TABLE test_tab_partition_f (a int primary key);
+ALTER TABLE test_tab_partition ADD CONSTRAINT test_tab_patition_fk FOREIGN KEY(a) REFERENCES test_tab_partition_f(a);
+});
+
+# Check the subscriber log from now on.
+$offset = -s $node_subscriber->logfile;
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO test_tab_partitioned SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i)"
+);
+
+$node_subscriber->wait_for_log(
+ qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
+ $offset);
+
+# Drop the foreign key constraint on the subscriber, now it works.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
+
+$node_publisher->wait_for_catchup($appname);
+
+$result =
+ $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
+is($result, qq(5000),
+ 'data replicated to subscriber after dropping the foreign key');
+
+$node_subscriber->stop;
+$node_publisher->stop;
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 4137dc77b4..697e6a7ba3 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1891,6 +1891,7 @@ PageXLogRecPtr
PagetableEntry
Pairs
ParallelAppendState
+ParallelApplySafety
ParallelBitmapHeapState
ParallelBlockTableScanDesc
ParallelBlockTableScanWorker
--
2.23.0.windows.1
[application/octet-stream] v19-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch (29.0K, ../../OS3PR01MB62752B29CEC2592E18236A9A9E909@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-v19-0004-Retry-to-apply-streaming-xact-only-in-apply-work.patch)
download | inline diff:
From 4037ce8edf0d126f7087fc8f069c0ca12f29f70e Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 15 Jun 2022 11:02:08 +0800
Subject: [PATCH v19 4/4] Retry to apply streaming xact only in apply worker
When the subscription parameter is set streaming=parallel, the logic tries to
apply the streaming transaction using an apply background worker. If this
fails the background worker exits with an error.
In this case, retry applying the streaming transaction using the normal
streaming=on mode. This is done to avoid getting caught in a loop of the same
retry errors.
A new flag field "subretry" has been introduced to catalog "pg_subscription".
If the subscriber exits with an error, this flag will be set true, and
whenever the transaction is applied successfully, this flag is reset false.
Now, when deciding how to apply a streaming transaction, the logic can know if
this transaction has previously failed or not (by checking the "subretry"
field).
---
doc/src/sgml/catalogs.sgml | 9 +
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/catalog/pg_subscription.c | 1 +
src/backend/catalog/system_views.sql | 2 +-
src/backend/commands/subscriptioncmds.c | 1 +
.../replication/logical/applybgworker.c | 16 +-
src/backend/replication/logical/worker.c | 165 +++++++++++++-----
src/bin/pg_dump/pg_dump.c | 5 +-
src/include/catalog/pg_subscription.h | 4 +
.../subscription/t/032_streaming_apply.pl | 154 +++++++++-------
10 files changed, 253 insertions(+), 109 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 099b3c9661..1adfed667d 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7907,6 +7907,15 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>subretry</structfield> <type>bool</type>
+ </para>
+ <para>
+ True if the previous apply change failed, necessitating a retry
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>subconninfo</structfield> <type>text</type>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 832899570f..8ba6470e8a 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -244,6 +244,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
column in the relation on the subscriber-side should also be the
unique column on the publisher-side; 2) there cannot be any
non-immutable functions used by the subscriber-side replicated table.
+ When applying a streaming transaction, if either requirement is not
+ met, the background worker will exit with an error.
+ <literal>parallel</literal> mode is disregarded when retrying;
+ instead the transaction will be applied using <literal>on</literal>
+ mode.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index 33ae3da8ae..9faeb2b1c0 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok)
sub->stream = subform->substream;
sub->twophasestate = subform->subtwophasestate;
sub->disableonerr = subform->subdisableonerr;
+ sub->retry = subform->subretry;
/* Get conninfo */
datum = SysCacheGetAttr(SUBSCRIPTIONOID,
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index f369b1fc14..8284cdd00c 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1299,7 +1299,7 @@ REVOKE ALL ON pg_replication_origin_status FROM public;
REVOKE ALL ON pg_subscription FROM public;
GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled,
subbinary, substream, subtwophasestate, subdisableonerr,
- subslotname, subsynccommit, subpublications, suborigin)
+ subretry, subslotname, subsynccommit, subpublications, suborigin)
ON pg_subscription TO public;
CREATE VIEW pg_stat_subscription_stats AS
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 9697128414..bd55c6ac4d 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -689,6 +689,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
LOGICALREP_TWOPHASE_STATE_PENDING :
LOGICALREP_TWOPHASE_STATE_DISABLED);
values[Anum_pg_subscription_subdisableonerr - 1] = BoolGetDatum(opts.disableonerr);
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(false);
values[Anum_pg_subscription_subconninfo - 1] =
CStringGetTextDatum(conninfo);
if (opts.slot_name)
diff --git a/src/backend/replication/logical/applybgworker.c b/src/backend/replication/logical/applybgworker.c
index 89c712f785..5d7b55e5ab 100644
--- a/src/backend/replication/logical/applybgworker.c
+++ b/src/backend/replication/logical/applybgworker.c
@@ -106,6 +106,18 @@ apply_bgworker_can_start(TransactionId xid)
if (!XLogRecPtrIsInvalid(MySubscription->skiplsn))
return false;
+ /*
+ * Don't use apply background workers for retries, because it is possible
+ * that the last time we tried to apply a transaction using an apply
+ * background worker the checks failed (see function
+ * apply_bgworker_relation_check).
+ */
+ if (MySubscription->retry)
+ {
+ elog(DEBUG1, "apply background workers are not used for retries");
+ return false;
+ }
+
/*
* For streaming transactions that are being applied in apply background
* worker, we cannot decide whether to apply the change for a relation
@@ -840,7 +852,5 @@ apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
rel->remoterel.nspname, rel->remoterel.relname),
errdetail("The unique column on subscriber is not the unique "
"column on publisher or there is at least one "
- "non-immutable function."),
- errhint("Please change to use subscription parameter "
- "streaming=on.")));
+ "non-immutable function.")));
}
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 41793d26c2..c914cece99 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -379,6 +379,8 @@ static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
static inline void reset_apply_error_context_info(void);
+static void set_subscription_retry(bool retry);
+
/*
* Should this worker apply changes for given relation.
*
@@ -905,6 +907,9 @@ apply_handle_commit(StringInfo s)
apply_handle_commit_internal(&commit_data);
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1016,6 +1021,9 @@ apply_handle_prepare(StringInfo s)
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Check the status of apply background worker if any. */
apply_bgworker_check_status();
@@ -1069,6 +1077,9 @@ apply_handle_commit_prepared(StringInfo s)
store_flush_position(prepare_data.end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(prepare_data.end_lsn);
@@ -1124,6 +1135,9 @@ apply_handle_rollback_prepared(StringInfo s)
store_flush_position(rollback_data.rollback_end_lsn);
in_remote_transaction = false;
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
+
/* Process any tables that are being synchronized in parallel. */
process_syncing_tables(rollback_data.rollback_end_lsn);
@@ -1218,6 +1232,9 @@ apply_handle_stream_prepare(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
in_remote_transaction = false;
@@ -1646,6 +1663,9 @@ apply_handle_stream_abort(StringInfo s)
*/
serialize_stream_abort(xid, subxid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
reset_apply_error_context_info();
@@ -1858,6 +1878,9 @@ apply_handle_stream_commit(StringInfo s)
/* Unlink the files with serialized changes and subxact info. */
stream_cleanup_files(MyLogicalRepWorker->subid, xid);
}
+
+ /* Reset the retry flag. */
+ set_subscription_retry(false);
}
/* Check the status of apply background worker if any. */
@@ -3902,20 +3925,28 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed during table synchronization */
+ pgstat_report_subscription_error(MySubscription->oid, false);
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed during table synchronization. Abort
- * the current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, false);
- PG_RE_THROW();
- }
+ proc_exit(0);
}
PG_END_TRY();
@@ -3940,20 +3971,27 @@ start_apply(XLogRecPtr origin_startpos)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed while applying changes */
+ pgstat_report_subscription_error(MySubscription->oid,
+ !am_tablesync_worker());
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed while applying changes. Abort the
- * current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
-
- PG_RE_THROW();
- }
}
PG_END_TRY();
}
@@ -4200,28 +4238,11 @@ ApplyWorkerMain(Datum main_arg)
}
/*
- * After error recovery, disable the subscription in a new transaction
- * and exit cleanly.
+ * Disable the subscription in a new transaction.
*/
static void
DisableSubscriptionAndExit(void)
{
- /*
- * Emit the error message, and recover from the error state to an idle
- * state
- */
- HOLD_INTERRUPTS();
-
- EmitErrorReport();
- AbortOutOfAnyTransaction();
- FlushErrorState();
-
- RESUME_INTERRUPTS();
-
- /* Report the worker failed during either table synchronization or apply */
- pgstat_report_subscription_error(MyLogicalRepWorker->subid,
- !am_tablesync_worker());
-
/* Disable the subscription */
StartTransactionCommand();
DisableSubscription(MySubscription->oid);
@@ -4231,8 +4252,6 @@ DisableSubscriptionAndExit(void)
ereport(LOG,
errmsg("logical replication subscription \"%s\" has been disabled due to an error",
MySubscription->name));
-
- proc_exit(0);
}
/*
@@ -4467,3 +4486,63 @@ reset_apply_error_context_info(void)
apply_error_callback_arg.remote_attnum = -1;
set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
}
+
+/*
+ * Set subretry of pg_subscription catalog.
+ *
+ * If retry is true, subscriber is about to exit with an error. Otherwise, it
+ * means that the transaction was applied successfully.
+ */
+static void
+set_subscription_retry(bool retry)
+{
+ Relation rel;
+ HeapTuple tup;
+ bool started_tx = false;
+ bool nulls[Natts_pg_subscription];
+ bool replaces[Natts_pg_subscription];
+ Datum values[Natts_pg_subscription];
+
+ if (MySubscription->retry == retry ||
+ am_apply_bgworker())
+ return;
+
+ if (!IsTransactionState())
+ {
+ StartTransactionCommand();
+ started_tx = true;
+ }
+
+ /* Look up the subscription in the catalog */
+ rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+ tup = SearchSysCacheCopy1(SUBSCRIPTIONOID,
+ ObjectIdGetDatum(MySubscription->oid));
+
+ if (!HeapTupleIsValid(tup))
+ elog(ERROR, "subscription \"%s\" does not exist", MySubscription->name);
+
+ LockSharedObject(SubscriptionRelationId, MySubscription->oid, 0,
+ AccessShareLock);
+
+ /* Form a new tuple. */
+ memset(values, 0, sizeof(values));
+ memset(nulls, false, sizeof(nulls));
+ memset(replaces, false, sizeof(replaces));
+
+ /* Reset subretry */
+ values[Anum_pg_subscription_subretry - 1] = BoolGetDatum(retry);
+ replaces[Anum_pg_subscription_subretry - 1] = true;
+
+ tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+ replaces);
+
+ /* Update the catalog. */
+ CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+ /* Cleanup. */
+ heap_freetuple(tup);
+ table_close(rel, NoLock);
+
+ if (started_tx)
+ CommitTransactionCommand();
+}
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index b894cca929..e93959f17d 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4484,8 +4484,9 @@ getSubscriptions(Archive *fout)
ntups = PQntuples(res);
/*
- * Get subscription fields. We don't include subskiplsn in the dump as
- * after restoring the dump this value may no longer be relevant.
+ * Get subscription fields. We don't include subskiplsn and subretry in
+ * the dump as after restoring the dump this value may no longer be
+ * relevant.
*/
i_tableoid = PQfnumber(res, "tableoid");
i_oid = PQfnumber(res, "oid");
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 71ad03a934..9b72f4b40c 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -88,6 +88,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
bool subdisableonerr; /* True if a worker error should cause the
* subscription to be disabled */
+ bool subretry BKI_DEFAULT(f); /* True if the previous apply change failed. */
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* Connection string to the publisher */
text subconninfo BKI_FORCE_NOT_NULL;
@@ -131,6 +133,8 @@ typedef struct Subscription
bool disableonerr; /* Indicates if the subscription should be
* automatically disabled if a worker error
* occurs */
+ bool retry; /* Indicates if the previous apply change
+ * failed. */
char *conninfo; /* Connection string to the publisher */
char *slotname; /* Name of the replication slot */
char *synccommit; /* Synchronous commit setting for worker */
diff --git a/src/test/subscription/t/032_streaming_apply.pl b/src/test/subscription/t/032_streaming_apply.pl
index 7f8bfa6745..24c519a870 100644
--- a/src/test/subscription/t/032_streaming_apply.pl
+++ b/src/test/subscription/t/032_streaming_apply.pl
@@ -57,8 +57,13 @@ $node_subscriber->safe_psql(
$node_publisher->wait_for_catchup($appname);
+# ============================================================================
# It is not allowed that the unique index on the publisher and the subscriber
-# is different. Check the error reported by background worker in this case.
+# is different. Check the error reported by background worker in this case. And
+# after retrying in apply worker, we check if the data is replicated
+# successfully.
+# ============================================================================
+
# First we check the unique index on normal table.
$node_subscriber->safe_psql('postgres',
"CREATE UNIQUE INDEX test_tab_b_idx ON test_tab (b)");
@@ -71,14 +76,15 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
my $result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_idx");
# Then we check the unique index on partition table.
$node_subscriber->safe_psql('postgres',
@@ -95,17 +101,20 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the unique index on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(5000), 'data replicated to subscriber after dropping index');
+is($result, qq(5000), 'data replicated to subscribers after retrying because of unique index');
+
+# Drop the unique index on the subscriber.
+$node_subscriber->safe_psql('postgres', "DROP INDEX test_tab_b_partition_idx");
# Triggers which execute non-immutable function are not allowed on the
# subscriber side. Check the error reported by background worker in this case.
+# And after retrying in apply worker, we check if the data is replicated
+# successfully.
# First we check the trigger function on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -129,15 +138,16 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab");
# Then we check the trigger function on partition table.
$node_subscriber->safe_psql(
@@ -157,19 +167,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the trigger on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "DROP TRIGGER insert_trig ON test_tab_partition");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
-is($result, qq(0), 'data replicated to subscriber after dropping trigger');
+is($result, qq(0), 'data replicated to subscribers after retrying because of trigger');
+# Drop the trigger on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "DROP TRIGGER insert_trig ON test_tab_partition");
+
+# ============================================================================
# It is not allowed that column default value expression contains a
# non-immutable function on the subscriber side. Check the error reported by
-# background worker in this case.
+# background worker in this case. And after retrying in apply worker, we check
+# if the data is replicated successfully.
+# ============================================================================
+
# First we check the column default value expression on normal table.
$node_subscriber->safe_psql('postgres',
"ALTER TABLE test_tab ALTER COLUMN b SET DEFAULT random()");
@@ -185,16 +200,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN b DROP DEFAULT");
# Then we check the column default value expression on partition table.
$node_subscriber->safe_psql('postgres',
@@ -211,20 +227,25 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop default value on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping default value expression');
+ 'data replicated to subscribers after retrying because of column default value');
+
+# Drop default value on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition ALTER COLUMN b DROP DEFAULT");
+# ============================================================================
# It is not allowed that domain constraint expression contains a non-immutable
# function on the subscriber side. Check the error reported by background
-# worker in this case.
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# Because the column type of the partition table must be the same as its parent
# table, only test normal table here.
$node_subscriber->safe_psql(
@@ -242,21 +263,26 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop domain constraint expression on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(0),
- 'data replicated to subscriber after dropping domain constraint expression'
+ 'data replicated to subscribers after retrying because of domain'
);
-# It is not allowed that constraint expression contains a non-immutable function
-# on the subscriber side. Check the error reported by background worker in this
-# case.
+# Drop domain constraint expression on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab ALTER COLUMN a TYPE int");
+
+# ============================================================================
+# It is not allowed that constraint expression contains a non-immutable
+# function on the subscriber side. Check the error reported by background
+# worker in this case. And after retrying in apply worker, we check if the data
+# is replicated successfully.
+# ============================================================================
+
# First we check the constraint expression on normal table.
$node_subscriber->safe_psql(
'postgres', qq{
@@ -274,16 +300,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tab_con");
# Then we check the constraint expression on partition table.
$node_subscriber->safe_psql(
@@ -300,19 +327,24 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(0),
- 'data replicated to subscriber after dropping constraint expression');
+ 'data replicated to subscribers after retrying because of constraint');
+# Drop constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_con");
+
+# ============================================================================
# It is not allowed that foreign key on the subscriber side. Check the error
-# reported by background worker in this case.
+# reported by background worker in this case. And after retrying in apply
+# worker, we check if the data is replicated successfully.
+# ============================================================================
+
# First we check the foreign key on normal table.
$node_publisher->safe_psql('postgres', "DELETE FROM test_tab");
$node_publisher->wait_for_catchup($appname);
@@ -333,16 +365,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab" using subscription parameter streaming=parallel/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab DROP CONSTRAINT test_tabfk");
# Then we check the foreign key on partition table.
$node_publisher->wait_for_catchup($appname);
@@ -363,16 +396,17 @@ $node_subscriber->wait_for_log(
qr/ERROR: ( [A-Z0-9]+:)? cannot replicate target relation "public.test_tab_partitioned" using subscription parameter streaming=parallel/,
$offset);
-# Drop the foreign key constraint on the subscriber, now it works.
-$node_subscriber->safe_psql('postgres',
- "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
-
+# Wait for this streaming transaction to be applied in the apply worker.
$node_publisher->wait_for_catchup($appname);
$result =
$node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tab_partitioned");
is($result, qq(5000),
- 'data replicated to subscriber after dropping the foreign key');
+ 'data replicated to subscribers after retrying because of foreign key');
+
+# Drop the foreign key constraint on the subscriber.
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE test_tab_partition DROP CONSTRAINT test_tab_patition_fk");
$node_subscriber->stop;
$node_publisher->stop;
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-25 13:50 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 0 replies; 43+ messages in thread
From: Amit Kapila @ 2022-07-25 13:50 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 22, 2022 at 8:26 AM [email protected]
<[email protected]> wrote:
>
> On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > Attach the news patches.
>
> Not able to apply patches cleanly because the change in HEAD (366283961a).
> Therefore, I rebased the patch based on the changes in HEAD.
>
> Attach the new patches.
>
Few comments on 0001:
======================
1.
- <structfield>substream</structfield> <type>bool</type>
+ <structfield>substream</structfield> <type>char</type>
</para>
<para>
- If true, the subscription will allow streaming of in-progress
- transactions
+ Controls how to handle the streaming of in-progress transactions:
+ <literal>f</literal> = disallow streaming of in-progress transactions,
+ <literal>t</literal> = spill the changes of in-progress transactions to
+ disk and apply at once after the transaction is committed on the
+ publisher,
+ <literal>p</literal> = apply changes directly using a background worker
Shouldn't the description of 'p' be something like: apply changes
directly using a background worker, if available, otherwise, it
behaves the same as 't'
2.
Note that if an error happens when
+ applying changes in a background worker, the finish LSN of the
+ remote transaction might not be reported in the server log.
Is there any case where finish LSN can be reported when applying via
background worker, if not, then we should use 'won't' instead of
'might not'?
3.
+#define PG_LOGICAL_APPLY_SHM_MAGIC 0x79fb2447 // TODO Consider change
It is better to change this as the same magic number is used by
PG_TEST_SHM_MQ_MAGIC
4.
+ /* Ignore statistics fields that have been updated. */
+ s.cursor += IGNORE_SIZE_IN_MESSAGE;
Can we change the comment to: "Ignore statistics fields that have been
updated by the main apply worker."? Will it be better to name the
define as "SIZE_STATS_MESSAGE"?
5.
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *shared)
{
...
...
+ apply_dispatch(&s);
+
+ if (ConfigReloadPending)
+ {
+ ConfigReloadPending = false;
+ ProcessConfigFile(PGC_SIGHUP);
+ }
+
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
We should not process the config file under ApplyMessageContext. You
should switch context before processing the config file. See other
similar usages in the code.
6.
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *shared)
{
...
...
+ MemoryContextSwitchTo(oldctx);
+ MemoryContextReset(ApplyMessageContext);
+ }
+
+ MemoryContextSwitchTo(TopMemoryContext);
+ MemoryContextReset(ApplyContext);
...
}
I don't see the need to reset ApplyContext here as we don't do
anything in that context here.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-26 09:00 Dilip Kumar <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 2 replies; 43+ messages in thread
From: Dilip Kumar @ 2022-07-26 09:00 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jul 22, 2022 at 8:27 AM [email protected]
<[email protected]> wrote:
>
> On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > Attach the news patches.
>
> Not able to apply patches cleanly because the change in HEAD (366283961a).
> Therefore, I rebased the patch based on the changes in HEAD.
>
> Attach the new patches.
+ /* Check the foreign keys. */
+ fkeys = RelationGetFKeyList(entry->localrel);
+ if (fkeys)
+ entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
So if there is a foreign key on any of the tables which are parts of a
subscription then we do not allow changes for that subscription to be
applied in parallel? I think this is a big limitation because having
foreign key on the table is very normal right? I agree that if we
allow them then there could be failure due to out of order apply
right? but IMHO we should not put the restriction instead let it fail
if there is ever such conflict. Because if there is a conflict the
transaction will be sent again. Do we see that there could be wrong
or inconsistent results if we allow such things to be executed in
parallel. If not then IMHO just to avoid some corner case failure we
are restricting very normal cases.
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-26 09:33 Dilip Kumar <[email protected]>
parent: Dilip Kumar <[email protected]>
1 sibling, 1 reply; 43+ messages in thread
From: Dilip Kumar @ 2022-07-26 09:33 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jul 26, 2022 at 2:30 PM Dilip Kumar <[email protected]> wrote:
>
> On Fri, Jul 22, 2022 at 8:27 AM [email protected]
> <[email protected]> wrote:
> >
> > On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > > Attach the news patches.
> >
> > Not able to apply patches cleanly because the change in HEAD (366283961a).
> > Therefore, I rebased the patch based on the changes in HEAD.
> >
> > Attach the new patches.
>
> + /* Check the foreign keys. */
> + fkeys = RelationGetFKeyList(entry->localrel);
> + if (fkeys)
> + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
>
> So if there is a foreign key on any of the tables which are parts of a
> subscription then we do not allow changes for that subscription to be
> applied in parallel? I think this is a big limitation because having
> foreign key on the table is very normal right? I agree that if we
> allow them then there could be failure due to out of order apply
> right? but IMHO we should not put the restriction instead let it fail
> if there is ever such conflict. Because if there is a conflict the
> transaction will be sent again. Do we see that there could be wrong
> or inconsistent results if we allow such things to be executed in
> parallel. If not then IMHO just to avoid some corner case failure we
> are restricting very normal cases.
some more comments..
1.
+ /*
+ * If we have found a free worker or if we are already
applying this
+ * transaction in an apply background worker, then we
pass the data to
+ * that worker.
+ */
+ if (first_segment)
+ apply_bgworker_send_data(stream_apply_worker, s->len, s->data);
Comment says that if we have found a free worker or we are already
applying in the worker then pass the changes to the worker
but actually as per the code here we are only passing in case of first_segment?
I think what you are trying to say is that if it is first segment then send the
2.
+ /*
+ * This is the main apply worker. Check if there is any free apply
+ * background worker we can use to process this transaction.
+ */
+ if (first_segment)
+ stream_apply_worker = apply_bgworker_start(stream_xid);
+ else
+ stream_apply_worker = apply_bgworker_find(stream_xid);
So currently, whenever we get a new streamed transaction we try to
start a new background worker for that. Why do we need to start/close
the background apply worker every time we get a new streamed
transaction. I mean we can keep the worker in the pool for time being
and if there is a new transaction looking for a worker then we can
find from that. Starting a worker is costly operation and since we
are using parallelism for this mean we are expecting that there would
be frequent streamed transaction needing parallel apply worker so why
not to let it wait for a certain amount of time so that if load is low
it will anyway stop and if the load is high it will be reused for next
streamed transaction.
3.
Why are we restricting parallel apply workers only for the streamed
transactions, because streaming depends upon the size of the logical
decoding work mem so making steaming and parallel apply tightly
coupled seems too restrictive to me. Do we see some obvious problems
in applying other transactions in parallel?
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-26 09:56 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 0 replies; 43+ messages in thread
From: Peter Smith @ 2022-07-26 09:56 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
Here are some review comment for patch v19-0001:
======
1.1 Missing docs for protocol version
Since you bumped the logical replication protocol version for this
patch, now there is some missing documentation to describe this new
protocol version. e.g. See here [1]
======
1.2 doc/src/sgml/config.sgml
+ <para>
+ Maximum number of apply background workers per subscription. This
+ parameter controls the amount of parallelism of the streaming of
+ in-progress transactions when subscription parameter
+ <literal>streaming = parallel</literal>.
+ </para>
SUGGESTION
Maximum number of apply background workers per subscription. This
parameter controls the amount of parallelism for streaming of
in-progress transactions with subscription parameter
<literal>streaming = parallel</literal>.
======
1.3 src/sgml/protocol.sgml
@@ -6809,6 +6809,25 @@ psql "dbname=postgres replication=database" -c
"IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term>Int64 (XLogRecPtr)</term>
+ <listitem>
+ <para>
+ The LSN of the abort.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>Int64 (TimestampTz)</term>
+ <listitem>
+ <para>
+ Abort timestamp of the transaction. The value is in number
+ of microseconds since PostgreSQL epoch (2000-01-01).
+ </para>
+ </listitem>
+ </varlistentry>
There are missing notes on these new fields. They both should says
something like "This field is available since protocol version 4."
(See similar examples on the same docs page)
======
1.4 src/backend/replication/logical/applybgworker.c - apply_bgworker_start
Previously [1] I wrote:
> The Assert that the entries in the free-list are FINISHED seems like
> unnecessary checking. IIUC, code is already doing the Assert that
> entries are FINISHED before allowing them into the free-list in the
> first place.
IMO this Assert just causes unnecessary doubts, but if you really want
to keep it then I think it belongs logically *above* the
list_delete_last.
~~~
1.5 src/backend/replication/logical/applybgworker.c - apply_bgworker_start
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ wstate->shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
It makes no sense to assign a protocol version to a server_version.
Perhaps it is just a simple matter of a poorly named struct member.
e.g Maybe everything is OK if it said something like
wstate->shared->proto_version.
~~~
1.6 src/backend/replication/logical/applybgworker.c - LogicalApplyBgwLoop
+/* Apply Background Worker main loop */
+static void
+LogicalApplyBgwLoop(shm_mq_handle *mqh, volatile ApplyBgworkerShared *shared)
'shared' seems a very vague param name. Maybe can be 'bgw_shared' or
'parallel_shared' or something better?
~~~
1.7 src/backend/replication/logical/applybgworker.c - ApplyBgworkerMain
+/*
+ * Apply Background Worker entry point
+ */
+void
+ApplyBgworkerMain(Datum main_arg)
+{
+ volatile ApplyBgworkerShared *shared;
'shared' seems a very vague var name. Maybe can be 'bgw_shared' or
'parallel_shared' or something better?
~~~
1.8 src/backend/replication/logical/applybgworker.c - apply_bgworker_setup_dsm
+static void
+apply_bgworker_setup_dsm(ApplyBgworkerState *wstate)
+{
+ shm_toc_estimator e;
+ Size segsize;
+ dsm_segment *seg;
+ shm_toc *toc;
+ ApplyBgworkerShared *shared;
+ shm_mq *mq;
'shared' seems a very vague var name. Maybe can be 'bgw_shared' or
'parallel_shared' or something better?
~~~
1.9 src/backend/replication/logical/applybgworker.c - apply_bgworker_setup_dsm
+ server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+ shared->server_version =
+ server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+ server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+ server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+ LOGICALREP_PROTO_VERSION_NUM;
Same as earlier review comment #1.5
======
1.10 src/backend/replication/logical/worker.c
@@ -22,8 +22,28 @@
* STREAMED TRANSACTIONS
* ---------------------
* Streamed transactions (large transactions exceeding a memory limit on the
- * upstream) are not applied immediately, but instead, the data is written
- * to temporary files and then applied at once when the final commit arrives.
+ * upstream) are applied using one of two approaches.
+ *
+ * 1) Separate background workers
"two approaches." --> "two approaches:"
~~~
1.11 src/backend/replication/logical/worker.c - apply_handle_stream_abort
+ /* Check whether the publisher sends abort_lsn and abort_time. */
+ if (am_apply_bgworker())
+ read_abort_lsn = MyParallelShared->server_version >=
+ LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM;
IMO makes no sense to compare a server version with a protocol
version. Same as review comment #1.5
======
1.12 src/include/replication/worker_internal.h
+typedef struct ApplyBgworkerShared
+{
+ slock_t mutex;
+
+ /* Status of apply background worker. */
+ ApplyBgworkerStatus status;
+
+ /* server version of publisher. */
+ uint32 server_version;
+
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
AFAICT you only ever used 'server_version' for storing the *protocol*
version, so really this member should be called something like
'proto_version'. Please see earlier review comment #1.5 and others.
------
[1] https://www.postgresql.org/message-id/CAHut%2BPvN7fwtUE%3DbidzrsOUXSt%2BJpnkJztZ-Jn5t86moofaZ6g%40ma...
[2] https://www.postgresql.org/docs/devel/protocol-logical-replication.html
Kind Reagrds,
Peter Smith.
Fujitsu Australia.
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 03:37 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 0 replies; 43+ messages in thread
From: Peter Smith @ 2022-07-27 03:37 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
Here are some review comments for patch v19-0003:
======
3.1 doc/src/sgml/ref/create_subscription.sgml
@@ -240,6 +240,10 @@ CREATE SUBSCRIPTION <replaceable
class="parameter">subscription_name</replaceabl
transaction is committed. Note that if an error happens when
applying changes in a background worker, the finish LSN of the
remote transaction might not be reported in the server log.
+ <literal>parallel</literal> mode has two requirements: 1) the unique
+ column in the relation on the subscriber-side should also be the
+ unique column on the publisher-side; 2) there cannot be any
+ non-immutable functions used by the subscriber-side replicated table.
</para>
3.1a.
It looked a bit strange starting the sentence with the enum
"<literal>parallel</literal> mode". Maybe reword it something like:
"This mode has two requirements: ..."
or
"There are two requirements for using <literal>parallel</literal> mode: ..."
3.1b.
Point 1) says "relation", but point 2) says "table". I think the
consistent term should be used.
======
3.2 <general>
For consistency, please search all this patch and replace every:
"... applied by an apply background worker" -> "... applied using an
apply background worker"
And also search/replace every:
"... in the apply background worker: -> "... using an apply background worker"
======
3.3 .../replication/logical/applybgworker.c
@@ -800,3 +800,47 @@ apply_bgworker_subxact_info_add(TransactionId current_xid)
MemoryContextSwitchTo(oldctx);
}
}
+
+/*
+ * Check if changes on this relation can be applied by an apply background
+ * worker.
+ *
+ * Although the commit order is maintained only allowing one process to commit
+ * at a time, the access order to the relation has changed. This could cause
+ * unexpected problems if the unique column on the replicated table is
+ * inconsistent with the publisher-side or contains non-immutable functions
+ * when applying transactions in the apply background worker.
+ */
+void
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
"only allowing" -> "by only allowing" (I think you mean this, right?)
~~~
3.4
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel_apply == PARALLEL_APPLY_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot replicate target relation \"%s.%s\" using "
+ "subscription parameter streaming=parallel",
+ rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change to use subscription parameter "
+ "streaming=on.")));
3.4a.
Of course, the code should give the user the "correct error" if there
is an error (!), but having a comment explicitly saying so does not
serve any purpose.
3.4b.
The logic might be simplified if it was written differently like:
+ if (rel->parallel_apply != PARALLEL_APPLY_SAFE)
+ ereport(ERROR, ...
======
3.5 src/backend/replication/logical/proto.c
@@ -40,6 +41,68 @@ static void logicalrep_read_tuple(StringInfo in,
LogicalRepTupleData *tuple);
static void logicalrep_write_namespace(StringInfo out, Oid nspid);
static const char *logicalrep_read_namespace(StringInfo in);
+static Bitmapset *RelationGetUniqueKeyBitmap(Relation rel);
+
+/*
+ * RelationGetUniqueKeyBitmap -- get a bitmap of unique attribute numbers
+ *
+ * This is similar to RelationGetIdentityKeyBitmap(), but returns a bitmap of
+ * index attribute numbers for all unique indexes.
+ */
+static Bitmapset *
+RelationGetUniqueKeyBitmap(Relation rel)
Why is the forward declaration needed when the static function
immediately follows it?
======
3.6 src/backend/replication/logical/relation.c -
logicalrep_relmap_reset_parallel_cb
@@ -91,6 +98,26 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
}
}
+/*
+ * Relcache invalidation callback to reset parallel flag.
+ */
+static void
+logicalrep_relmap_reset_parallel_cb(Datum arg, int cacheid, uint32 hashvalue)
"reset parallel flag" -> "reset parallel_apply flag"
~~~
3.7 src/backend/replication/logical/relation.c -
logicalrep_rel_mark_parallel_apply
+ * There are two requirements for applying changes in an apply background
+ * worker: 1) The unique column in the relation on the subscriber-side should
+ * also be the unique column on the publisher-side; 2) There cannot be any
+ * non-immutable functions used by the subscriber-side.
This comment should exactly match the help text. See review comment #3.1
~~~
3.8
+ /* Initialize the flag. */
+ entry->parallel_apply = PARALLEL_APPLY_SAFE;
I previously suggested [1] (#3.6b) to move this. Consider, that you
cannot logically flag the entry as "safe" until you are certain that
it is safe. And you cannot be sure of that until you've passed all the
checks this function is doing. Therefore IMO the assignment to
PARALLEL_APPLY_SAFE should be the last line of the function.
~~~
3.9
+ /*
+ * Then, check if there is any non-immutable function used by the local
+ * table. Look for functions in the following places:
+ * a. trigger functions;
+ * b. Column default value expressions and domain constraints;
+ * c. Constraint expressions;
+ * d. Foreign keys.
+ */
"used by the local table" -> "used by the subscriber-side relation"
(reworded so that it is consistent with the First comment)
~~~
3.10
I previously suggested [1] (#3.7) to use goto in this function to
avoid the excessive number of returns. IMO there is nothing inherently
evil about gotos, so long as they are used with care - sometimes they
are the best option. Anyway, I attached some BEFORE/AFTER example code
to this post - others can judge which way is preferable.
======
3.11 src/backend/utils/cache/typcache.c - GetDomainConstraints
@@ -2540,6 +2540,23 @@ compare_values_of_enum(TypeCacheEntry *tcache,
Oid arg1, Oid arg2)
return 0;
}
+/*
+ * GetDomainConstraints --- get DomainConstraintState list of
specified domain type
+ */
+List *
+GetDomainConstraints(Oid type_id)
+{
+ TypeCacheEntry *typentry;
+ List *constraints = NIL;
+
+ typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
+
+ if(typentry->domainData != NULL)
+ constraints = typentry->domainData->constraints;
+
+ return constraints;
+}
This function can be simplified (if you want). e.g.
List *
GetDomainConstraints(Oid type_id)
{
TypeCacheEntry *typentry;
typentry = lookup_type_cache(type_id, TYPECACHE_DOMAIN_CONSTR_INFO);
return typentry->domainData ? typentry->domainData->constraints : NIL;
}
======
3.12 src/include/replication/logicalrelation.h
@@ -15,6 +15,19 @@
#include "access/attmap.h"
#include "replication/logicalproto.h"
+/*
+ * States to determine if changes on one relation can be applied using an
+ * apply background worker.
+ */
+typedef enum ParalleApplySafety
+{
+ PARALLEL_APPLY_UNKNOWN = 0, /* unknown */
+ PARALLEL_APPLY_SAFE, /* Can apply changes in an apply background
+ worker */
+ PARALLEL_APPLY_UNSAFE /* Can not apply changes in an apply background
+ worker */
+} ParalleApplySafety;
+
3.12a
Typo in enum and typedef names:
"ParalleApplySafety" -> "ParallelApplySafety"
3.12b
I think the values are quite self-explanatory now. Commenting on each
of them separately is not really adding anything useful.
3.12c.
New enum missing from typedefs.list?
======
3.13 typdefs.list
Should include the new typedef. See comment #3.12c.
------
[1] https://www.postgresql.org/message-id/OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9%40OS3PR01MB6275.jpnprd0...
Kind Regards,
Peter Smith.
Fujitsu Australia
/*
* Check if changes on one relation can be applied by an apply background
* worker and assign the 'parallel_apply' flag.
*
* There are two requirements for applying changes in an apply background
* worker: 1) The unique column in the relation on the subscriber-side should
* also be the unique column on the publisher-side; 2) There cannot be any
* non-immutable functions used by the subscriber-side.
*
* We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
* on one relation can not be applied by an apply background worker and leave
* it to apply_bgworker_relation_check() to throw the actual error if needed.
*/
static void
logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
{
Bitmapset *ukey;
int i;
TupleDesc tupdesc;
int attnum;
List *fkeys = NIL;
/* Fast path if 'parallel_apply' flag is already known. */
if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
return;
/*
* First, check if the unique column in the relation on the subscriber-side
* is also the unique column on the publisher-side.
*/
ukey = RelationGetIndexAttrBitmap(entry->localrel,
INDEX_ATTR_BITMAP_KEY);
if (ukey)
{
i = -1;
while ((i = bms_next_member(ukey, i)) >= 0)
{
attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
if (entry->attrmap->attnums[attnum] < 0 ||
!bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
goto parallel_apply_unsafe;
}
bms_free(ukey);
}
/*
* Then, check if there is any non-immutable function used by the local
* table. Look for functions in the following places:
* a. trigger functions;
* b. Column default value expressions and domain constraints;
* c. Constraint expressions;
* d. Foreign keys.
*/
/* Check the trigger functions. */
if (entry->localrel->trigdesc != NULL)
{
for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
{
Trigger *trig = entry->localrel->trigdesc->triggers + i;
if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
continue;
if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
goto parallel_apply_unsafe;
}
}
/* Check the columns. */
tupdesc = RelationGetDescr(entry->localrel);
for (attnum = 0; attnum < tupdesc->natts; attnum++)
{
Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
/* We don't need info for dropped or generated attributes */
if (att->attisdropped || att->attgenerated)
continue;
/*
* We don't need to check columns that only exist on the
* subscriber
*/
if (entry->attrmap->attnums[attnum] < 0)
continue;
if (att->atthasdef)
{
Node *defaultexpr;
defaultexpr = build_column_default(entry->localrel, attnum + 1);
if (contain_mutable_functions(defaultexpr))
goto parallel_apply_unsafe;
}
/*
* If the column is of a DOMAIN type, determine whether
* that domain has any CHECK expressions that are not
* immutable.
*/
if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
{
List *domain_constraints;
ListCell *lc;
domain_constraints = GetDomainConstraints(att->atttypid);
foreach(lc, domain_constraints)
{
DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
goto parallel_apply_unsafe;
}
}
}
/* Check the constraints. */
if (tupdesc->constr)
{
ConstrCheck *check = tupdesc->constr->check;
/*
* Determine if there are any CHECK constraints which
* contains non-immutable function.
*/
for (i = 0; i < tupdesc->constr->num_check; i++)
{
Expr *check_expr = stringToNode(check[i].ccbin);
if (contain_mutable_functions((Node *) check_expr))
goto parallel_apply_unsafe;
}
}
/* Check the foreign keys. */
fkeys = RelationGetFKeyList(entry->localrel);
if (fkeys)
goto parallel_apply_unsafe;
parallel_apply_safe:
entry->parallel_apply = PARALLEL_APPLY_SAFE;
return;
parallel_apply_unsafe:
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
/*
* Check if changes on one relation can be applied by an apply background
* worker and assign the 'parallel_apply' flag.
*
* There are two requirements for applying changes in an apply background
* worker: 1) The unique column in the relation on the subscriber-side should
* also be the unique column on the publisher-side; 2) There cannot be any
* non-immutable functions used by the subscriber-side.
*
* We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
* on one relation can not be applied by an apply background worker and leave
* it to apply_bgworker_relation_check() to throw the actual error if needed.
*/
static void
logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
{
Bitmapset *ukey;
int i;
TupleDesc tupdesc;
int attnum;
List *fkeys = NIL;
/* Fast path if 'parallel_apply' flag is already known. */
if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
return;
/* Initialize the flag. */
entry->parallel_apply = PARALLEL_APPLY_SAFE;
/*
* First, check if the unique column in the relation on the subscriber-side
* is also the unique column on the publisher-side.
*/
ukey = RelationGetIndexAttrBitmap(entry->localrel,
INDEX_ATTR_BITMAP_KEY);
if (ukey)
{
i = -1;
while ((i = bms_next_member(ukey, i)) >= 0)
{
attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
if (entry->attrmap->attnums[attnum] < 0 ||
!bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
bms_free(ukey);
}
/*
* Then, check if there is any non-immutable function used by the local
* table. Look for functions in the following places:
* a. trigger functions;
* b. Column default value expressions and domain constraints;
* c. Constraint expressions;
* d. Foreign keys.
*/
/* Check the trigger functions. */
if (entry->localrel->trigdesc != NULL)
{
for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
{
Trigger *trig = entry->localrel->trigdesc->triggers + i;
if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
continue;
if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
/* Check the columns. */
tupdesc = RelationGetDescr(entry->localrel);
for (attnum = 0; attnum < tupdesc->natts; attnum++)
{
Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
/* We don't need info for dropped or generated attributes */
if (att->attisdropped || att->attgenerated)
continue;
/*
* We don't need to check columns that only exist on the
* subscriber
*/
if (entry->attrmap->attnums[attnum] < 0)
continue;
if (att->atthasdef)
{
Node *defaultexpr;
defaultexpr = build_column_default(entry->localrel, attnum + 1);
if (contain_mutable_functions(defaultexpr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
/*
* If the column is of a DOMAIN type, determine whether
* that domain has any CHECK expressions that are not
* immutable.
*/
if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
{
List *domain_constraints;
ListCell *lc;
domain_constraints = GetDomainConstraints(att->atttypid);
foreach(lc, domain_constraints)
{
DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
}
/* Check the constraints. */
if (tupdesc->constr)
{
ConstrCheck *check = tupdesc->constr->check;
/*
* Determine if there are any CHECK constraints which
* contains non-immutable function.
*/
for (i = 0; i < tupdesc->constr->num_check; i++)
{
Expr *check_expr = stringToNode(check[i].ccbin);
if (contain_mutable_functions((Node *) check_expr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
/* Check the foreign keys. */
fkeys = RelationGetFKeyList(entry->localrel);
if (fkeys)
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
}
Attachments:
[text/plain] logicalrep_rel_mark_parallel_apply-with-goto.txt (4.2K, ../../CAHut+Pv9cKurDQHtk-ygYp45-8LYdE=4sMZY-8UmbeDTGgECVg@mail.gmail.com/2-logicalrep_rel_mark_parallel_apply-with-goto.txt)
download | inline:
/*
* Check if changes on one relation can be applied by an apply background
* worker and assign the 'parallel_apply' flag.
*
* There are two requirements for applying changes in an apply background
* worker: 1) The unique column in the relation on the subscriber-side should
* also be the unique column on the publisher-side; 2) There cannot be any
* non-immutable functions used by the subscriber-side.
*
* We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
* on one relation can not be applied by an apply background worker and leave
* it to apply_bgworker_relation_check() to throw the actual error if needed.
*/
static void
logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
{
Bitmapset *ukey;
int i;
TupleDesc tupdesc;
int attnum;
List *fkeys = NIL;
/* Fast path if 'parallel_apply' flag is already known. */
if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
return;
/*
* First, check if the unique column in the relation on the subscriber-side
* is also the unique column on the publisher-side.
*/
ukey = RelationGetIndexAttrBitmap(entry->localrel,
INDEX_ATTR_BITMAP_KEY);
if (ukey)
{
i = -1;
while ((i = bms_next_member(ukey, i)) >= 0)
{
attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
if (entry->attrmap->attnums[attnum] < 0 ||
!bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
goto parallel_apply_unsafe;
}
bms_free(ukey);
}
/*
* Then, check if there is any non-immutable function used by the local
* table. Look for functions in the following places:
* a. trigger functions;
* b. Column default value expressions and domain constraints;
* c. Constraint expressions;
* d. Foreign keys.
*/
/* Check the trigger functions. */
if (entry->localrel->trigdesc != NULL)
{
for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
{
Trigger *trig = entry->localrel->trigdesc->triggers + i;
if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
continue;
if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
goto parallel_apply_unsafe;
}
}
/* Check the columns. */
tupdesc = RelationGetDescr(entry->localrel);
for (attnum = 0; attnum < tupdesc->natts; attnum++)
{
Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
/* We don't need info for dropped or generated attributes */
if (att->attisdropped || att->attgenerated)
continue;
/*
* We don't need to check columns that only exist on the
* subscriber
*/
if (entry->attrmap->attnums[attnum] < 0)
continue;
if (att->atthasdef)
{
Node *defaultexpr;
defaultexpr = build_column_default(entry->localrel, attnum + 1);
if (contain_mutable_functions(defaultexpr))
goto parallel_apply_unsafe;
}
/*
* If the column is of a DOMAIN type, determine whether
* that domain has any CHECK expressions that are not
* immutable.
*/
if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
{
List *domain_constraints;
ListCell *lc;
domain_constraints = GetDomainConstraints(att->atttypid);
foreach(lc, domain_constraints)
{
DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
goto parallel_apply_unsafe;
}
}
}
/* Check the constraints. */
if (tupdesc->constr)
{
ConstrCheck *check = tupdesc->constr->check;
/*
* Determine if there are any CHECK constraints which
* contains non-immutable function.
*/
for (i = 0; i < tupdesc->constr->num_check; i++)
{
Expr *check_expr = stringToNode(check[i].ccbin);
if (contain_mutable_functions((Node *) check_expr))
goto parallel_apply_unsafe;
}
}
/* Check the foreign keys. */
fkeys = RelationGetFKeyList(entry->localrel);
if (fkeys)
goto parallel_apply_unsafe;
parallel_apply_safe:
entry->parallel_apply = PARALLEL_APPLY_SAFE;
return;
parallel_apply_unsafe:
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
[text/plain] logicalrep_rel_mark_parallel_apply-without-goto.txt (4.3K, ../../CAHut+Pv9cKurDQHtk-ygYp45-8LYdE=4sMZY-8UmbeDTGgECVg@mail.gmail.com/3-logicalrep_rel_mark_parallel_apply-without-goto.txt)
download | inline:
/*
* Check if changes on one relation can be applied by an apply background
* worker and assign the 'parallel_apply' flag.
*
* There are two requirements for applying changes in an apply background
* worker: 1) The unique column in the relation on the subscriber-side should
* also be the unique column on the publisher-side; 2) There cannot be any
* non-immutable functions used by the subscriber-side.
*
* We just mark the relation entry as 'PARALLEL_APPLY_UNSAFE' here if changes
* on one relation can not be applied by an apply background worker and leave
* it to apply_bgworker_relation_check() to throw the actual error if needed.
*/
static void
logicalrep_rel_mark_parallel_apply(LogicalRepRelMapEntry *entry)
{
Bitmapset *ukey;
int i;
TupleDesc tupdesc;
int attnum;
List *fkeys = NIL;
/* Fast path if 'parallel_apply' flag is already known. */
if (entry->parallel_apply != PARALLEL_APPLY_UNKNOWN)
return;
/* Initialize the flag. */
entry->parallel_apply = PARALLEL_APPLY_SAFE;
/*
* First, check if the unique column in the relation on the subscriber-side
* is also the unique column on the publisher-side.
*/
ukey = RelationGetIndexAttrBitmap(entry->localrel,
INDEX_ATTR_BITMAP_KEY);
if (ukey)
{
i = -1;
while ((i = bms_next_member(ukey, i)) >= 0)
{
attnum = AttrNumberGetAttrOffset(i + FirstLowInvalidHeapAttributeNumber);
if (entry->attrmap->attnums[attnum] < 0 ||
!bms_is_member(entry->attrmap->attnums[attnum], entry->remoterel.attunique))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
bms_free(ukey);
}
/*
* Then, check if there is any non-immutable function used by the local
* table. Look for functions in the following places:
* a. trigger functions;
* b. Column default value expressions and domain constraints;
* c. Constraint expressions;
* d. Foreign keys.
*/
/* Check the trigger functions. */
if (entry->localrel->trigdesc != NULL)
{
for (i = 0; i < entry->localrel->trigdesc->numtriggers; i++)
{
Trigger *trig = entry->localrel->trigdesc->triggers + i;
if (trig->tgenabled != TRIGGER_FIRES_ALWAYS &&
trig->tgenabled != TRIGGER_FIRES_ON_REPLICA)
continue;
if (func_volatile(trig->tgfoid) != PROVOLATILE_IMMUTABLE)
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
/* Check the columns. */
tupdesc = RelationGetDescr(entry->localrel);
for (attnum = 0; attnum < tupdesc->natts; attnum++)
{
Form_pg_attribute att = TupleDescAttr(tupdesc, attnum);
/* We don't need info for dropped or generated attributes */
if (att->attisdropped || att->attgenerated)
continue;
/*
* We don't need to check columns that only exist on the
* subscriber
*/
if (entry->attrmap->attnums[attnum] < 0)
continue;
if (att->atthasdef)
{
Node *defaultexpr;
defaultexpr = build_column_default(entry->localrel, attnum + 1);
if (contain_mutable_functions(defaultexpr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
/*
* If the column is of a DOMAIN type, determine whether
* that domain has any CHECK expressions that are not
* immutable.
*/
if (get_typtype(att->atttypid) == TYPTYPE_DOMAIN)
{
List *domain_constraints;
ListCell *lc;
domain_constraints = GetDomainConstraints(att->atttypid);
foreach(lc, domain_constraints)
{
DomainConstraintState *con = (DomainConstraintState *) lfirst(lc);
if (con->check_expr && contain_mutable_functions((Node *) con->check_expr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
}
/* Check the constraints. */
if (tupdesc->constr)
{
ConstrCheck *check = tupdesc->constr->check;
/*
* Determine if there are any CHECK constraints which
* contains non-immutable function.
*/
for (i = 0; i < tupdesc->constr->num_check; i++)
{
Expr *check_expr = stringToNode(check[i].ccbin);
if (contain_mutable_functions((Node *) check_expr))
{
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
return;
}
}
}
/* Check the foreign keys. */
fkeys = RelationGetFKeyList(entry->localrel);
if (fkeys)
entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
}
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 04:36 Amit Kapila <[email protected]>
parent: Dilip Kumar <[email protected]>
1 sibling, 1 reply; 43+ messages in thread
From: Amit Kapila @ 2022-07-27 04:36 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; +Cc: [email protected] <[email protected]>; Peter Smith <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jul 26, 2022 at 2:30 PM Dilip Kumar <[email protected]> wrote:
>
> On Fri, Jul 22, 2022 at 8:27 AM [email protected]
> <[email protected]> wrote:
> >
> > On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > > Attach the news patches.
> >
> > Not able to apply patches cleanly because the change in HEAD (366283961a).
> > Therefore, I rebased the patch based on the changes in HEAD.
> >
> > Attach the new patches.
>
> + /* Check the foreign keys. */
> + fkeys = RelationGetFKeyList(entry->localrel);
> + if (fkeys)
> + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
>
> So if there is a foreign key on any of the tables which are parts of a
> subscription then we do not allow changes for that subscription to be
> applied in parallel?
>
I think the above check will just prevent the parallelism for a xact
operating on the corresponding relation not the relations of the
entire subscription. Your statement sounds like you are saying that it
will prevent parallelism for all the other tables in the subscription
which has a table with FK.
> I think this is a big limitation because having
> foreign key on the table is very normal right? I agree that if we
> allow them then there could be failure due to out of order apply
> right?
>
What kind of failure do you have in mind and how it can occur? The one
way it can fail is if the publisher doesn't have a corresponding
foreign key on the table because then the publisher could have allowed
an insert into a table (insert into FK table without having the
corresponding key in PK table) which may not be allowed on the
subscriber. However, I don't see any check that could prevent this
because for this we need to compare the FK list for a table from the
publisher with the corresponding one on the subscriber. I am not
really sure if due to the risk of such conflicts we should block
parallelism of transactions operating on tables with FK because those
conflicts can occur even without parallelism, it is just a matter of
timing. But, I could be missing something due to which the above check
can be useful?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 05:28 Dilip Kumar <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 43+ messages in thread
From: Dilip Kumar @ 2022-07-27 05:28 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Peter Smith <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jul 27, 2022 at 10:06 AM Amit Kapila <[email protected]> wrote:
>
> On Tue, Jul 26, 2022 at 2:30 PM Dilip Kumar <[email protected]> wrote:
> >
> > On Fri, Jul 22, 2022 at 8:27 AM [email protected]
> > <[email protected]> wrote:
> > >
> > > On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > > > Attach the news patches.
> > >
> > > Not able to apply patches cleanly because the change in HEAD (366283961a).
> > > Therefore, I rebased the patch based on the changes in HEAD.
> > >
> > > Attach the new patches.
> >
> > + /* Check the foreign keys. */
> > + fkeys = RelationGetFKeyList(entry->localrel);
> > + if (fkeys)
> > + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
> >
> > So if there is a foreign key on any of the tables which are parts of a
> > subscription then we do not allow changes for that subscription to be
> > applied in parallel?
> >
>
> I think the above check will just prevent the parallelism for a xact
> operating on the corresponding relation not the relations of the
> entire subscription. Your statement sounds like you are saying that it
> will prevent parallelism for all the other tables in the subscription
> which has a table with FK.
Okay, got it. I thought we are disallowing parallelism for the entire
subscription.
> > I think this is a big limitation because having
> > foreign key on the table is very normal right? I agree that if we
> > allow them then there could be failure due to out of order apply
> > right?
> >
>
> What kind of failure do you have in mind and how it can occur? The one
> way it can fail is if the publisher doesn't have a corresponding
> foreign key on the table because then the publisher could have allowed
> an insert into a table (insert into FK table without having the
> corresponding key in PK table) which may not be allowed on the
> subscriber. However, I don't see any check that could prevent this
> because for this we need to compare the FK list for a table from the
> publisher with the corresponding one on the subscriber. I am not
> really sure if due to the risk of such conflicts we should block
> parallelism of transactions operating on tables with FK because those
> conflicts can occur even without parallelism, it is just a matter of
> timing. But, I could be missing something due to which the above check
> can be useful?
Actually, my question starts with this check[1][2], from this it
appears that if this relation is having a foreign key then we are
marking it parallel unsafe[2] and later in [1] while the worker is
applying changes for that relation and if it was marked parallel
unsafe then we are throwing error. So my question was why we are
putting this restriction? Although this error is only talking about
unique and non-immutable functions this is also giving an error if the
target table had a foreign key. So my question was do we really need
to restrict this? I mean why we are restricting this case?
[1]
+apply_bgworker_relation_check(LogicalRepRelMapEntry *rel)
+{
+ /* Skip check if not an apply background worker. */
+ if (!am_apply_bgworker())
+ return;
+
+ /*
+ * Partition table checks are done later in function
+ * apply_handle_tuple_routing.
+ */
+ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ return;
+
+ /*
+ * Return if changes on this relation can be applied by an apply background
+ * worker.
+ */
+ if (rel->parallel_apply == PARALLEL_APPLY_SAFE)
+ return;
+
+ /* We are in error mode and should give user correct error. */
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot replicate target relation \"%s.%s\" using "
+ "subscription parameter streaming=parallel",
+ rel->remoterel.nspname, rel->remoterel.relname),
+ errdetail("The unique column on subscriber is not the unique "
+ "column on publisher or there is at least one "
+ "non-immutable function."),
+ errhint("Please change to use subscription parameter "
+ "streaming=on.")));
+}
[2]
> > + /* Check the foreign keys. */
> > + fkeys = RelationGetFKeyList(entry->localrel);
> > + if (fkeys)
> > + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 07:57 [email protected] <[email protected]>
parent: Dilip Kumar <[email protected]>
0 siblings, 1 reply; 43+ messages in thread
From: [email protected] @ 2022-07-27 07:57 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Peter Smith <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wednesday, July 27, 2022 1:29 PM Dilip Kumar <[email protected]> wrote:
>
> On Wed, Jul 27, 2022 at 10:06 AM Amit Kapila <[email protected]>
> wrote:
> >
> > On Tue, Jul 26, 2022 at 2:30 PM Dilip Kumar <[email protected]> wrote:
> > >
> > > On Fri, Jul 22, 2022 at 8:27 AM [email protected]
> > > <[email protected]> wrote:
> > > >
> > > > On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > > > > Attach the news patches.
> > > >
> > > > Not able to apply patches cleanly because the change in HEAD
> (366283961a).
> > > > Therefore, I rebased the patch based on the changes in HEAD.
> > > >
> > > > Attach the new patches.
> > >
> > > + /* Check the foreign keys. */
> > > + fkeys = RelationGetFKeyList(entry->localrel);
> > > + if (fkeys)
> > > + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
> > >
> > > So if there is a foreign key on any of the tables which are parts of
> > > a subscription then we do not allow changes for that subscription to
> > > be applied in parallel?
> > >
> >
> > I think the above check will just prevent the parallelism for a xact
> > operating on the corresponding relation not the relations of the
> > entire subscription. Your statement sounds like you are saying that it
> > will prevent parallelism for all the other tables in the subscription
> > which has a table with FK.
>
> Okay, got it. I thought we are disallowing parallelism for the entire subscription.
>
> > > I think this is a big limitation because having foreign key on the
> > > table is very normal right? I agree that if we allow them then
> > > there could be failure due to out of order apply right?
> > >
> >
> > What kind of failure do you have in mind and how it can occur? The one
> > way it can fail is if the publisher doesn't have a corresponding
> > foreign key on the table because then the publisher could have allowed
> > an insert into a table (insert into FK table without having the
> > corresponding key in PK table) which may not be allowed on the
> > subscriber. However, I don't see any check that could prevent this
> > because for this we need to compare the FK list for a table from the
> > publisher with the corresponding one on the subscriber. I am not
> > really sure if due to the risk of such conflicts we should block
> > parallelism of transactions operating on tables with FK because those
> > conflicts can occur even without parallelism, it is just a matter of
> > timing. But, I could be missing something due to which the above check
> > can be useful?
>
> Actually, my question starts with this check[1][2], from this it
> appears that if this relation is having a foreign key then we are
> marking it parallel unsafe[2] and later in [1] while the worker is
> applying changes for that relation and if it was marked parallel
> unsafe then we are throwing error. So my question was why we are
> putting this restriction? Although this error is only talking about
> unique and non-immutable functions this is also giving an error if the
> target table had a foreign key. So my question was do we really need
> to restrict this? I mean why we are restricting this case?
>
Hi,
I think the foreign key check is used to prevent the apply worker from waiting
indefinitely which is caused by foreign key difference between publisher and
subscriber, Like the following example:
-------------------------------------
Publisher:
-- both table are published
CREATE TABLE PKTABLE ( ptest1 int);
CREATE TABLE FKTABLE ( ftest1 int);
-- initial data
INSERT INTO PKTABLE VALUES(1);
Subcriber:
CREATE TABLE PKTABLE ( ptest1 int PRIMARY KEY);
CREATE TABLE FKTABLE ( ftest1 int REFERENCES PKTABLE);
-- Execute the following transactions on publisher
Tx1:
INSERT ... -- make enough changes to start streaming mode
DELETE FROM PKTABLE;
Tx2:
INSERT ITNO FKTABLE VALUES(1);
COMMIT;
COMMIT;
-------------------------------------
The subcriber's apply worker will wait indefinitely, because the main apply worker is
waiting for the streaming transaction to finish which is in another apply
bgworker.
BTW, I think the foreign key won't take effect in subscriber's apply worker by
default. Because we set session_replication_role to 'replica' in apply worker
which prevent the FK trigger function to be executed(only the trigger with
FIRES_ON_REPLICA flag will be executed in this mode). User can only alter the
trigger to enable it on replica mode to make the foreign key work. So, ISTM, we
won't hit this ERROR frequently.
And based on this, another comment about the patch is that it seems unnecessary
to directly check the FK returned by RelationGetFKeyList. Checking the actual FK
trigger function seems enough.
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 08:03 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 0 replies; 43+ messages in thread
From: Peter Smith @ 2022-07-27 08:03 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
Here are some review comments for the patch v19-0004:
======
1. doc/src/sgml/ref/create_subscription.sgml
@@ -244,6 +244,11 @@ CREATE SUBSCRIPTION <replaceable
class="parameter">subscription_name</replaceabl
column in the relation on the subscriber-side should also be the
unique column on the publisher-side; 2) there cannot be any
non-immutable functions used by the subscriber-side replicated table.
+ When applying a streaming transaction, if either requirement is not
+ met, the background worker will exit with an error.
+ <literal>parallel</literal> mode is disregarded when retrying;
+ instead the transaction will be applied using <literal>on</literal>
+ mode.
</para>
That last sentence starting with lowercase seems odd - that's why I
thought saying "The parallel mode..." might be better. IMO "on mode"
seems strange too. Hence my previous [1] (#4.3) suggestion for this
SUGGESTION
The <literal>parallel</literal> mode is disregarded when retrying;
instead the transaction will be applied using <literal>streaming =
on</literal>.
======
2. src/backend/replication/logical/worker.c - start_table_sync
@@ -3902,20 +3925,28 @@ start_table_sync(XLogRecPtr *origin_startpos,
char **myslotname)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed during table synchronization */
+ pgstat_report_subscription_error(MySubscription->oid, false);
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed during table synchronization. Abort
- * the current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, false);
- PG_RE_THROW();
- }
+ proc_exit(0);
}
But is it correct to set the 'retry' flag even if the
MySubscription->disableonerr is true? Won’t that mean even after the
user corrects the problem and then re-enabled the subscription it
still won't let the streaming=parallel work, because that retry flag
is set?
Also, Something seems wrong to me here - IIUC the patch changed this
code because of the potential risk of an error within the
set_subscription_retry function, but now if such an error happens the
current code will bypass even getting to DisableSubscriptionAndExit,
so the subscription won't have a chance to get disabled as the user
might have wanted.
~~~
3. src/backend/replication/logical/worker.c - start_apply
@@ -3940,20 +3971,27 @@ start_apply(XLogRecPtr origin_startpos)
}
PG_CATCH();
{
+ /*
+ * Emit the error message, and recover from the error state to an idle
+ * state
+ */
+ HOLD_INTERRUPTS();
+
+ EmitErrorReport();
+ AbortOutOfAnyTransaction();
+ FlushErrorState();
+
+ RESUME_INTERRUPTS();
+
+ /* Report the worker failed while applying changes */
+ pgstat_report_subscription_error(MySubscription->oid,
+ !am_tablesync_worker());
+
+ /* Set the retry flag. */
+ set_subscription_retry(true);
+
if (MySubscription->disableonerr)
DisableSubscriptionAndExit();
- else
- {
- /*
- * Report the worker failed while applying changes. Abort the
- * current transaction so that the stats message is sent in an
- * idle state.
- */
- AbortOutOfAnyTransaction();
- pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
-
- PG_RE_THROW();
- }
}
(Same as previous review comment #2)
But is it correct to set the 'retry' flag even if the
MySubscription->disableonerr is true? Won’t that mean even after the
user corrects the problem and then re-enabled the subscription it
still won't let the streaming=parallel work, because that retry flag
is set?
Also, Something seems wrong to me here - IIUC the patch changed this
code because of the potential risk of an error within the
set_subscription_retry function, but now if such an error happens the
current code will bypass even getting to DisableSubscriptionAndExit,
so the subscription won't have a chance to get disabled as the user
might have wanted.
~~~
4. src/backend/replication/logical/worker.c - DisableSubscriptionAndExit
/*
- * After error recovery, disable the subscription in a new transaction
- * and exit cleanly.
+ * Disable the subscription in a new transaction.
*/
static void
DisableSubscriptionAndExit(void)
{
- /*
- * Emit the error message, and recover from the error state to an idle
- * state
- */
- HOLD_INTERRUPTS();
-
- EmitErrorReport();
- AbortOutOfAnyTransaction();
- FlushErrorState();
-
- RESUME_INTERRUPTS();
-
- /* Report the worker failed during either table synchronization or apply */
- pgstat_report_subscription_error(MyLogicalRepWorker->subid,
- !am_tablesync_worker());
-
/* Disable the subscription */
StartTransactionCommand();
DisableSubscription(MySubscription->oid);
@@ -4231,8 +4252,6 @@ DisableSubscriptionAndExit(void)
ereport(LOG,
errmsg("logical replication subscription \"%s\" has been disabled
due to an error",
MySubscription->name));
-
- proc_exit(0);
}
4a.
Hmm, I think it is a bad idea to remove the "exiting" code from the
function but still leave the function name the same as before saying
"AndExit".
4b.
Also, now the patch is unconditionally doing proc_exit(0) in the
calling code where previously it would do PG_RE_THROW. So it's a
subtle difference from the path the code used to take for worker
errors..
~~~
5. src/backend/replication/logical/worker.c - set_subscription_retry
@@ -4467,3 +4486,63 @@ reset_apply_error_context_info(void)
apply_error_callback_arg.remote_attnum = -1;
set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
}
+
+/*
+ * Set subretry of pg_subscription catalog.
+ *
+ * If retry is true, subscriber is about to exit with an error. Otherwise, it
+ * means that the transaction was applied successfully.
+ */
+static void
+set_subscription_retry(bool retry)
+{
+ Relation rel;
+ HeapTuple tup;
+ bool started_tx = false;
+ bool nulls[Natts_pg_subscription];
+ bool replaces[Natts_pg_subscription];
+ Datum values[Natts_pg_subscription];
+
+ if (MySubscription->retry == retry ||
+ am_apply_bgworker())
+ return;
Currently, I think this new 'subretry' field is only used to decide
whether a retry can use an apply background worker or not. I think all
this logic is *only* used when streaming=parallel. But AFAICT the
logic for setting/clearing the retry flag is executed *always*
regardless of the streaming mode.
So for all the times when the user did not ask for streaming=parallel
doesn't this just cause unnecessary overhead for every transaction?
------
[1] https://www.postgresql.org/message-id/OS3PR01MB62758A6AAED27B3A848CEB7A9E8F9%40OS3PR01MB6275.jpnprd0...
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 08:21 [email protected] <[email protected]>
parent: Dilip Kumar <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: [email protected] @ 2022-07-27 08:21 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tuesday, July 26, 2022 5:34 PM Dilip Kumar <[email protected]> wrote:
> On Tue, Jul 26, 2022 at 2:30 PM Dilip Kumar <[email protected]> wrote:
> >
> > On Fri, Jul 22, 2022 at 8:27 AM [email protected]
> > <[email protected]> wrote:
> > >
> > > On Tues, Jul 19, 2022 at 10:29 AM I wrote:
> > > > Attach the news patches.
> > >
> > > Not able to apply patches cleanly because the change in HEAD
> (366283961a).
> > > Therefore, I rebased the patch based on the changes in HEAD.
> > >
> > > Attach the new patches.
> >
> > + /* Check the foreign keys. */
> > + fkeys = RelationGetFKeyList(entry->localrel);
> > + if (fkeys)
> > + entry->parallel_apply = PARALLEL_APPLY_UNSAFE;
> >
> > So if there is a foreign key on any of the tables which are parts of a
> > subscription then we do not allow changes for that subscription to be
> > applied in parallel? I think this is a big limitation because having
> > foreign key on the table is very normal right? I agree that if we
> > allow them then there could be failure due to out of order apply
> > right? but IMHO we should not put the restriction instead let it fail
> > if there is ever such conflict. Because if there is a conflict the
> > transaction will be sent again. Do we see that there could be wrong
> > or inconsistent results if we allow such things to be executed in
> > parallel. If not then IMHO just to avoid some corner case failure we
> > are restricting very normal cases.
>
> some more comments..
> 1.
> + /*
> + * If we have found a free worker or if we are already
> applying this
> + * transaction in an apply background worker, then we
> pass the data to
> + * that worker.
> + */
> + if (first_segment)
> + apply_bgworker_send_data(stream_apply_worker, s->len,
> + s->data);
>
> Comment says that if we have found a free worker or we are already applying in
> the worker then pass the changes to the worker but actually as per the code
> here we are only passing in case of first_segment?
>
> I think what you are trying to say is that if it is first segment then send the
>
> 2.
> + /*
> + * This is the main apply worker. Check if there is any free apply
> + * background worker we can use to process this transaction.
> + */
> + if (first_segment)
> + stream_apply_worker = apply_bgworker_start(stream_xid);
> + else
> + stream_apply_worker = apply_bgworker_find(stream_xid);
>
> So currently, whenever we get a new streamed transaction we try to start a new
> background worker for that. Why do we need to start/close the background
> apply worker every time we get a new streamed transaction. I mean we can
> keep the worker in the pool for time being and if there is a new transaction
> looking for a worker then we can find from that. Starting a worker is costly
> operation and since we are using parallelism for this mean we are expecting
> that there would be frequent streamed transaction needing parallel apply
> worker so why not to let it wait for a certain amount of time so that if load is low
> it will anyway stop and if the load is high it will be reused for next streamed
> transaction.
It seems the function name was a bit mislead. Currently, the started apply
bgworker won't exit after applying the transaction. And the
apply_bgworker_start will first try to choose a free worker. It will start a
new worker only if no free worker is available.
> 3.
> Why are we restricting parallel apply workers only for the streamed
> transactions, because streaming depends upon the size of the logical decoding
> work mem so making steaming and parallel apply tightly coupled seems too
> restrictive to me. Do we see some obvious problems in applying other
> transactions in parallel?
We thought there could be some conflict failure and deadlock if we parallel
apply normal transaction which need transaction dependency check[1]. But I will do
some more research for this and share the result soon.
[1] https://www.postgresql.org/message-id/CAA4eK1%2BwyN6zpaHUkCLorEWNx75MG0xhMwcFhvjqm2KURZEAGw%40mail.g...
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-27 12:41 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
5 siblings, 1 reply; 43+ messages in thread
From: [email protected] @ 2022-07-27 12:41 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>; Peter Smith <[email protected]>
Dear Wang-san,
Hi, I'm also interested in the patch and I started to review this.
Followings are comments about 0001.
1. terminology
In your patch a new worker "apply background worker" has been introduced,
but I thought it might be confused because PostgreSQL has already the worker "background worker".
Both of apply worker and apply bworker are categolized as bgworker.
Do you have any reasons not to use "apply parallel worker" or "apply streaming worker"?
(Note that I'm not native English speaker)
2. logicalrep_worker_stop()
```
- /* No worker, nothing to do. */
- if (!worker)
- {
- LWLockRelease(LogicalRepWorkerLock);
- return;
- }
+ if (worker)
+ logicalrep_worker_stop_internal(worker);
+
+ LWLockRelease(LogicalRepWorkerLock);
+}
```
I thought you could add a comment the meaning of if-statement, like "No main apply worker, nothing to do"
3. logicalrep_workers_find()
I thought you could add a description about difference between this and logicalrep_worker_find() at the top of the function.
IIUC logicalrep_workers_find() counts subworker, but logicalrep_worker_find() does not focus such type of workers.
4. logicalrep_worker_detach()
```
static void
logicalrep_worker_detach(void)
{
+ /*
+ * If we are the main apply worker, stop all the apply background workers
+ * we started before.
+ *
```
I thought "we are" should be "This is", based on other comments.
5. applybgworker.c
```
+/* Apply background workers hash table (initialized on first use) */
+static HTAB *ApplyWorkersHash = NULL;
+static List *ApplyWorkersFreeList = NIL;
+static List *ApplyWorkersList = NIL;
```
I thought they should be ApplyBgWorkersXXX, because they stores information only related with apply bgworkers.
6. ApplyBgworkerShared
```
+ TransactionId stream_xid;
+ uint32 n; /* id of apply background worker */
+} ApplyBgworkerShared;
```
I thought the field "n" is too general, how about "proc_id" or "worker_id"?
7. apply_bgworker_wait_for()
```
+ /* If any workers (or the postmaster) have died, we have failed. */
+ if (status == APPLY_BGWORKER_EXIT)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("background worker %u failed to apply transaction %u",
+ wstate->shared->n, wstate->shared->stream_xid)))
```
7.a
I thought we should not mention about PM death here, because in this case
apply worker will exit at WaitLatch().
7.b
The error message should be "apply background worker %u...".
8. apply_bgworker_check_status()
```
+ errmsg("background worker %u exited unexpectedly",
+ wstate->shared->n)));
```
The error message should be "apply background worker %u...".
9. apply_bgworker_send_data()
```
+ if (result != SHM_MQ_SUCCESS)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("could not send tuples to shared-memory queue")));
```
I thought the error message should be "could not send data to..."
because sent data might not be tuples. For example, in case of STEAM PREPARE, I thit does not contain tuple.
10. wait_event.h
```
WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT,
+ WAIT_EVENT_LOGICAL_APPLY_WORKER_STATE_CHANGE,
WAIT_EVENT_LOGICAL_SYNC_DATA,
```
I thought the event should be WAIT_EVENT_LOGICAL_APPLY_BG_WORKER_STATE_CHANGE,
because this is used when apply worker waits until the status of bgworker changes.
Best Regards,
Hayato Kuroda
FUJITSU LIMITED
^ permalink raw reply [nested|flat] 43+ messages in thread
* RE: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-28 05:20 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: [email protected] @ 2022-07-28 05:20 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>; Peter Smith <[email protected]>
Dear Wang,
I found further comments about the test code.
11. src/test/regress/sql/subscription.sql
```
-- fail - streaming must be boolean
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
```
The comment is no longer correct: should be "streaming must be boolean or 'parallel'"
12. src/test/regress/sql/subscription.sql
```
-- now it works
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
```
I think we should test the case of streaming = 'parallel'.
13. 015_stream.pl
I could not find test about TRUNCATE. IIUC apply bgworker works well
even if it gets LOGICAL_REP_MSG_TRUNCATE message from main worker.
Can you add the case?
Best Regards,
Hayato Kuroda
FUJITSU LIMITED
^ permalink raw reply [nested|flat] 43+ messages in thread
* Re: Perform streaming logical transactions by background workers and parallel apply
@ 2022-07-28 13:32 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 0 replies; 43+ messages in thread
From: Amit Kapila @ 2022-07-28 13:32 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Dilip Kumar <[email protected]>; [email protected] <[email protected]>; Peter Smith <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jul 27, 2022 at 1:27 PM [email protected]
<[email protected]> wrote:
>
> On Wednesday, July 27, 2022 1:29 PM Dilip Kumar <[email protected]> wrote:
> >
> > On Wed, Jul 27, 2022 at 10:06 AM Amit Kapila <[email protected]>
> > >
> > > What kind of failure do you have in mind and how it can occur? The one
> > > way it can fail is if the publisher doesn't have a corresponding
> > > foreign key on the table because then the publisher could have allowed
> > > an insert into a table (insert into FK table without having the
> > > corresponding key in PK table) which may not be allowed on the
> > > subscriber. However, I don't see any check that could prevent this
> > > because for this we need to compare the FK list for a table from the
> > > publisher with the corresponding one on the subscriber. I am not
> > > really sure if due to the risk of such conflicts we should block
> > > parallelism of transactions operating on tables with FK because those
> > > conflicts can occur even without parallelism, it is just a matter of
> > > timing. But, I could be missing something due to which the above check
> > > can be useful?
> >
> > Actually, my question starts with this check[1][2], from this it
> > appears that if this relation is having a foreign key then we are
> > marking it parallel unsafe[2] and later in [1] while the worker is
> > applying changes for that relation and if it was marked parallel
> > unsafe then we are throwing error. So my question was why we are
> > putting this restriction? Although this error is only talking about
> > unique and non-immutable functions this is also giving an error if the
> > target table had a foreign key. So my question was do we really need
> > to restrict this? I mean why we are restricting this case?
> >
>
> Hi,
>
> I think the foreign key check is used to prevent the apply worker from waiting
> indefinitely which is caused by foreign key difference between publisher and
> subscriber, Like the following example:
>
> -------------------------------------
> Publisher:
> -- both table are published
> CREATE TABLE PKTABLE ( ptest1 int);
> CREATE TABLE FKTABLE ( ftest1 int);
>
> -- initial data
> INSERT INTO PKTABLE VALUES(1);
>
> Subcriber:
> CREATE TABLE PKTABLE ( ptest1 int PRIMARY KEY);
> CREATE TABLE FKTABLE ( ftest1 int REFERENCES PKTABLE);
>
> -- Execute the following transactions on publisher
>
> Tx1:
> INSERT ... -- make enough changes to start streaming mode
> DELETE FROM PKTABLE;
> Tx2:
> INSERT ITNO FKTABLE VALUES(1);
> COMMIT;
> COMMIT;
> -------------------------------------
>
> The subcriber's apply worker will wait indefinitely, because the main apply worker is
> waiting for the streaming transaction to finish which is in another apply
> bgworker.
>
IIUC, here the problem will be that TX2 (Insert in FK table) performed
by the apply worker will wait for a parallel worker doing streaming
transaction TX1 which has performed Delete from PK table. This wait is
required because we can't decide if Insert will be successful or not
till TX1 is either committed or Rollback. This is similar to the
problem related to primary/unique keys mentioned earlier [1]. If so,
then, we should try to forbid this in some way to avoid subscribers
from being stuck.
Dilip, does this reason sounds sufficient to you for such a check, or
do you still think we don't need any check for FK's?
>
> BTW, I think the foreign key won't take effect in subscriber's apply worker by
> default. Because we set session_replication_role to 'replica' in apply worker
> which prevent the FK trigger function to be executed(only the trigger with
> FIRES_ON_REPLICA flag will be executed in this mode). User can only alter the
> trigger to enable it on replica mode to make the foreign key work. So, ISTM, we
> won't hit this ERROR frequently.
>
> And based on this, another comment about the patch is that it seems unnecessary
> to directly check the FK returned by RelationGetFKeyList. Checking the actual FK
> trigger function seems enough.
>
That is correct. I think it would have been better if we can detect
that publisher doesn't have FK but the subscriber has FK as it can
occur only in that scenario. If that requires us to send more
information from the publisher, we can leave it for now (as this
doesn't seem to be a frequent scenario) and keep a simpler check based
on subscriber schema.
I think we should add a test as mentioned by you above so that if
tomorrow one tries to remove the FK check, we have a way to know.
Also, please add comments and tests for additional checks related to
constraints in the patch.
[1] - https://www.postgresql.org/message-id/CAA4eK1JwahU_WuP3S%2B7POqta%3DPhm_3gxZeVmJuuoUq1NV%3DkrXA%40ma...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 43+ messages in thread
end of thread, other threads:[~2022-07-28 13:32 UTC | newest]
Thread overview: 43+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2021-02-03 15:40 [PATCH 1/1] Remove support for COPY FROM with protocol version 2. Heikki Linnakangas <[email protected]>
2022-07-01 06:43 Re: Perform streaming logical transactions by background workers and parallel apply Peter Smith <[email protected]>
2022-07-01 09:43 ` Re: Perform streaming logical transactions by background workers and parallel apply Amit Kapila <[email protected]>
2022-07-07 03:45 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-07 03:44 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-07 10:20 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-13 04:33 ` Re: Perform streaming logical transactions by background workers and parallel apply Peter Smith <[email protected]>
2022-07-19 02:28 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-22 02:56 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-25 13:50 ` Re: Perform streaming logical transactions by background workers and parallel apply Amit Kapila <[email protected]>
2022-07-26 09:00 ` Re: Perform streaming logical transactions by background workers and parallel apply Dilip Kumar <[email protected]>
2022-07-26 09:33 ` Re: Perform streaming logical transactions by background workers and parallel apply Dilip Kumar <[email protected]>
2022-07-27 08:21 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-27 04:36 ` Re: Perform streaming logical transactions by background workers and parallel apply Amit Kapila <[email protected]>
2022-07-27 05:28 ` Re: Perform streaming logical transactions by background workers and parallel apply Dilip Kumar <[email protected]>
2022-07-27 07:57 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-28 13:32 ` Re: Perform streaming logical transactions by background workers and parallel apply Amit Kapila <[email protected]>
2022-07-26 09:56 ` Re: Perform streaming logical transactions by background workers and parallel apply Peter Smith <[email protected]>
2022-07-27 03:37 ` Re: Perform streaming logical transactions by background workers and parallel apply Peter Smith <[email protected]>
2022-07-27 08:03 ` Re: Perform streaming logical transactions by background workers and parallel apply Peter Smith <[email protected]>
2022-07-27 12:41 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
2022-07-28 05:20 ` RE: Perform streaming logical transactions by background workers and parallel apply [email protected] <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox