public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. 49+ messages / 2 participants [nested] [flat]
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v3 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-04-01 20:22 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-04-01 20:22 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Reviewed-by: John Naylor Discussion: https://www.postgresql.org/message-id/89627a2a-c123-a8aa-267e-5d168feb73dd%40iki.fi --- src/backend/commands/copyfromparse.c | 111 +++++++++------------------ 1 file changed, 35 insertions(+), 76 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 0813424768..b52abc1520 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -90,21 +90,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - input_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -159,7 +144,7 @@ static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, - int minread, int maxread); + int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); @@ -230,9 +215,8 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* * CopyGetData reads data from the source (file or frontend) * - * We attempt to read at least minread, and at most maxread, bytes from - * the source. The actual number of bytes read is returned; if this is - * less than minread, EOF was detected. + * We attempt to read at maxread bytes from the source. The actual + * number of bytes read is returned; if this is 0, EOF was detected. * * Note: when copying from the frontend, we expect a proper EOF mark per * protocol; if the frontend simply drops the connection, we raise error. @@ -241,7 +225,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) * NB: no data conversion is applied here. */ static int -CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) +CopyGetData(CopyFromState cstate, void *databuf, int maxread) { int bytesread = 0; @@ -257,7 +241,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) cstate->raw_reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread == 0 && !cstate->raw_reached_eof) { int avail; @@ -321,7 +305,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) } break; case COPY_CALLBACK: - bytesread = cstate->data_source_cb(databuf, minread, maxread); + bytesread = cstate->data_source_cb(databuf, 1, maxread); + if (bytesread == 0) + cstate->raw_reached_eof = true; break; } @@ -605,7 +591,7 @@ CopyLoadRawBuf(CopyFromState cstate) /* Load more data */ inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, - 1, RAW_BUF_SIZE - cstate->raw_buf_len); + RAW_BUF_SIZE - cstate->raw_buf_len); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_len = nbytes; @@ -990,7 +976,7 @@ CopyReadLine(CopyFromState cstate) do { inbytes = CopyGetData(cstate, cstate->input_buf, - 1, INPUT_BUF_SIZE); + INPUT_BUF_SIZE); } while (inbytes > 0); cstate->input_buf_index = 0; cstate->input_buf_len = 0; @@ -1047,8 +1033,7 @@ CopyReadLineText(CopyFromState cstate) char *copy_input_buf; int input_buf_ptr; int copy_buf_len; - bool need_data = false; - bool hit_eof = false; + bool hit_eof; bool result = false; /* CSV variables */ @@ -1098,6 +1083,7 @@ CopyReadLineText(CopyFromState cstate) copy_input_buf = cstate->input_buf; input_buf_ptr = cstate->input_buf_index; copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; for (;;) { @@ -1105,35 +1091,37 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (input_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (input_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - CopyLoadInputBuf(cstate); - /* update our local variables */ - hit_eof = cstate->input_reached_eof; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + CopyLoadInputBuf(cstate); + /* update our local variables */ + hit_eof = cstate->input_reached_eof; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (INPUT_BUF_BYTES(cstate) <= 0) + if (copy_buf_len - input_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -1142,20 +1130,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -1189,14 +1163,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of input_buf. + * Look at the next character. If we're at EOF, c will wind + * up as '\0' because of the guaranteed pad of input_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_input_buf[input_buf_ptr]; if (c == '\n') @@ -1262,7 +1231,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -1277,16 +1245,9 @@ CopyReadLineText(CopyFromState cstate) { input_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 == '\n') @@ -1309,9 +1270,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 != '\r' && c2 != '\n') -- 2.30.2 --------------11E1AF537C24312AF77A7698-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* [PATCH v4 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-04-06 17:48 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Heikki Linnakangas @ 2021-04-06 17:48 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). FIXME: This fails with sequence "\.<NL><some invalid bytes>". We should detect the end-of-copy marker \. and stop reading without complaining about the garbage after the end-of-copy marker. That doesn't work if we force 4 bytes of lookahead; the invalid byte sequence fits in the lookahead window, so we will try to convert it. Reviewed-by: John Naylor Discussion: https://www.postgresql.org/message-id/89627a2a-c123-a8aa-267e-5d168feb73dd%40iki.fi --- src/backend/commands/copyfromparse.c | 112 +++++++++------------------ 1 file changed, 36 insertions(+), 76 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 0813424768..6b140531ed 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -90,21 +90,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - input_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -159,7 +144,7 @@ static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, - int minread, int maxread); + int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); @@ -230,9 +215,8 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* * CopyGetData reads data from the source (file or frontend) * - * We attempt to read at least minread, and at most maxread, bytes from - * the source. The actual number of bytes read is returned; if this is - * less than minread, EOF was detected. + * We attempt to read at most maxread bytes from the source. The actual + * number of bytes read is returned; if this is 0, EOF was detected. * * Note: when copying from the frontend, we expect a proper EOF mark per * protocol; if the frontend simply drops the connection, we raise error. @@ -241,7 +225,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) * NB: no data conversion is applied here. */ static int -CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) +CopyGetData(CopyFromState cstate, void *databuf, int maxread) { int bytesread = 0; @@ -257,7 +241,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) cstate->raw_reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread == 0 && !cstate->raw_reached_eof) { int avail; @@ -321,7 +305,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) } break; case COPY_CALLBACK: - bytesread = cstate->data_source_cb(databuf, minread, maxread); + bytesread = cstate->data_source_cb(databuf, 1, maxread); + if (bytesread == 0) + cstate->raw_reached_eof = true; break; } @@ -605,7 +591,7 @@ CopyLoadRawBuf(CopyFromState cstate) /* Load more data */ inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, - 1, RAW_BUF_SIZE - cstate->raw_buf_len); + RAW_BUF_SIZE - cstate->raw_buf_len); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_len = nbytes; @@ -990,7 +976,7 @@ CopyReadLine(CopyFromState cstate) do { inbytes = CopyGetData(cstate, cstate->input_buf, - 1, INPUT_BUF_SIZE); + INPUT_BUF_SIZE); } while (inbytes > 0); cstate->input_buf_index = 0; cstate->input_buf_len = 0; @@ -1047,8 +1033,7 @@ CopyReadLineText(CopyFromState cstate) char *copy_input_buf; int input_buf_ptr; int copy_buf_len; - bool need_data = false; - bool hit_eof = false; + bool hit_eof; bool result = false; /* CSV variables */ @@ -1098,6 +1083,7 @@ CopyReadLineText(CopyFromState cstate) copy_input_buf = cstate->input_buf; input_buf_ptr = cstate->input_buf_index; copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; for (;;) { @@ -1105,35 +1091,38 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead at most three bytes in one iteration of the + * loop (for the sequence \.<CR><NL>), so make sure we have at least + * four bytes in the buffer. Note that we always guarantee that there + * is one \0 in the buffer, after last valid byte; the lookaheads + * below rely on that. */ - if (input_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (input_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - CopyLoadInputBuf(cstate); - /* update our local variables */ - hit_eof = cstate->input_reached_eof; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + CopyLoadInputBuf(cstate); + /* update our local variables */ + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; + continue; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (INPUT_BUF_BYTES(cstate) <= 0) + if (copy_buf_len - input_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -1142,20 +1131,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -1189,14 +1164,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of input_buf. + * Look at the next character. If we're at EOF, c will wind + * up as '\0' because of the guaranteed pad of input_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_input_buf[input_buf_ptr]; if (c == '\n') @@ -1262,7 +1232,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -1277,16 +1246,9 @@ CopyReadLineText(CopyFromState cstate) { input_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 == '\n') @@ -1309,9 +1271,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 != '\r' && c2 != '\n') -- 2.30.2 --------------1601CF8503AB8D78BE138DD6-- ^ permalink raw reply [nested|flat] 49+ messages in thread
* Re: Report search_path value back to the client. @ 2024-07-18 20:47 Tomas Vondra <[email protected]> 0 siblings, 0 replies; 49+ messages in thread From: Tomas Vondra @ 2024-07-18 20:47 UTC (permalink / raw) To: Jelte Fennema-Nio <[email protected]>; Tom Lane <[email protected]>; +Cc: Alexey Klyukin <[email protected]>; Alexander Kukushkin <[email protected]>; pgsql-hackers On 11/3/23 10:06, Jelte Fennema-Nio wrote: > I wanted to revive this thread, since it's by far one of the most > common foot guns that people run into with PgBouncer. Almost all > session level SET commands leak across transactions, but SET > search_path is by far the one with the biggest impact when it is not > the setting that you expect. As well as being a very common setting to > change. In the Citus extension we actually change search_path to be > GUC_REPORT so that PgBouncer can track it, because otherwise Citus its > schema based sharding starts breaking completely when using PgBouncer. > [1] > > On Fri, 3 Nov 2023 at 09:38, Tom Lane <[email protected]> wrote: >> I'm against this on a couple of different grounds: >> >> 1. Performance. ... >> >> 2. Semantics. The existing GUC_REPORT variables are all things that >> directly relate to client-visible behavior, eg how values of type >> timestamp will be interpreted and printed. search_path is no such thing, > >> >> We could possibly alleviate problem #1 by changing the behavior of guc.c >> so it doesn't report every single transition of flagged variables, but >> only (say) once just before ReadyForQuery if the variable changed in >> the just-finished command. That's not exactly a one-line fix though. > > The proposed fix for #1 has been committed since PG14 in > 2432b1a04087edc2fd9536c7c9aa4ca03fd1b363 > > So that only leaves #2. I think search_path can arguably be considered > to be client visible, because it changes how the client and its > queries are parsed by Postgres. And even if you don't agree with that > argument, it's simply not true that the only GUCs that are GUC_REPORT > are related to client-visible behaviour. Things like application_name > and is_superuser are also GUC_REPORT [2]. > Not sure about whether search_path should be considered client-visible. It's true most GUCs either don't affect query execution, or affect how things are executed, but not what results are produced. Which search_path does, so in this sense it is "client visible". That being said, it this makes using pgbouncer easier (or even possible in some applications where it currently does not work), I'd vote to get this committed. >> so it's hard to make a principled argument for reporting it that doesn't >> lead to the conclusion that you want *everything* reported. (In >> particular, I don't believe at all your argument that this would help >> pgbouncer significantly.) > > To be clear, I would like it to be configurable which settings are > GUC_REPORT. Because there are others that are useful for PgBouncer to > track (e.g. statement_timeout and lock_timeout) That's why I've been > active on the thread proposing such a change [3]. But that thread has > not been getting a lot of attention, I guess because it's a large > change that includes protocol changes. So that's why I'm reviving this > thread again. Since search_path is by far the most painful setting for > PgBouncer users. A common foot-gun is that running pg_dump causes > breakage for other clients, because its "SET search_path" is leaked to > the next transaction [4]. > So, did that other patch move forward, in some way? The last message is from January, I'm not sure I want to read through that thread just to find out it's stuck on something. Also, I recall we had a session at pgconf.dev to discuss this protocol stuff. I don't remember what the conclusions from that part were :-( Stupid idea - could we have a GUC/function/something to define which GUCs the client wants to get reports for? Maybe that'd be simpler and would not require protocol changes? Not as pretty, of course, and maybe it has some fatal flaw. In any case, I think it'd be good to decide what to do with this patch. Whether to reject it or get it committed, even if we hope to get a better / extensible solution in the future. I'd vote to commit. What concerns me a little bit is if this will make our life more complicated in the future. I mean, imagine we get it committed, and then get the protocol stuff later. Wouldn't that mean pgbouncer needs to do three different things, depending on the server version? (without search_path reporting, with reporting and with the new protocol stuff?) regards -- Tomas Vondra EnterpriseDB: http://www.enterprisedb.com The Enterprise PostgreSQL Company ^ permalink raw reply [nested|flat] 49+ messages in thread
end of thread, other threads:[~2024-07-18 20:47 UTC | newest] Thread overview: 49+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-04-01 20:22 [PATCH v3 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-04-06 17:48 [PATCH v4 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2024-07-18 20:47 Re: Report search_path value back to the client. Tomas Vondra <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox