public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. 50+ messages / 2 participants [nested] [flat]
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. @ 2021-02-04 10:39 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-02-04 10:39 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index dd629668540..757b240a3c8 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += nbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1087,7 +1047,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.0 --------------AD4B53627CEDABF9E8CC5B3C-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-03-04 09:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-03-04 09:10 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Discussion: https://www.postgresql.org/message-id/9ec25819-0a8a-d51a-17dc-4150bb3cca3b%40iki.fi --- src/backend/commands/copyfromparse.c | 119 ++++++++--------------- src/include/commands/copyfrom_internal.h | 3 +- 2 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ce24a1528bd..c2efe7e0782 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -46,21 +46,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -118,7 +103,7 @@ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static bool CopyLoadRawBuf(CopyFromState cstate); +static bool CopyLoadRawBuf(CopyFromState cstate, int minread); static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void @@ -209,7 +194,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); - if (bytesread == 0) + if (bytesread < maxread) cstate->reached_eof = true; break; case COPY_FRONTEND: @@ -278,6 +263,8 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case COPY_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); + if (bytesread < minread) + cstate->reached_eof = true; break; } @@ -329,14 +316,13 @@ CopyGetInt16(CopyFromState cstate, int16 *val) /* * CopyLoadRawBuf loads some more data into raw_buf * - * Returns true if able to obtain at least one more byte, else false. + * Returns true if able to obtain at least 'minread' bytes, else false. * * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start - * of the buffer and then we load more data after that. This case occurs only - * when a multibyte character crosses a bufferload boundary. + * of the buffer and then we load more data after that. */ static bool -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromState cstate, int minread) { int nbytes = RAW_BUF_BYTES(cstate); int inbytes; @@ -347,14 +333,15 @@ CopyLoadRawBuf(CopyFromState cstate) nbytes); inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes, - 1, RAW_BUF_SIZE - nbytes); + minread, RAW_BUF_SIZE - nbytes); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_index = 0; cstate->raw_buf_len = nbytes; cstate->bytes_processed += inbytes; pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); - return (inbytes > 0); + + return (inbytes >= minread); } /* @@ -389,7 +376,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) /* Load more data if buffer is empty. */ if (RAW_BUF_BYTES(cstate) == 0) { - if (!CopyLoadRawBuf(cstate)) + if (!CopyLoadRawBuf(cstate, 1)) break; /* EOF */ } @@ -678,7 +665,7 @@ CopyReadLine(CopyFromState cstate) do { cstate->raw_buf_index = cstate->raw_buf_len; - } while (CopyLoadRawBuf(cstate)); + } while (CopyLoadRawBuf(cstate, 1)); } } else @@ -747,7 +734,6 @@ CopyReadLineText(CopyFromState cstate) char *copy_raw_buf; int raw_buf_ptr; int copy_buf_len; - bool need_data = false; bool hit_eof = false; bool result = false; char mblen_str[2]; @@ -794,6 +780,7 @@ CopyReadLineText(CopyFromState cstate) copy_raw_buf = cstate->raw_buf; raw_buf_ptr = cstate->raw_buf_index; copy_buf_len = cstate->raw_buf_len; + hit_eof = cstate->reached_eof; for (;;) { @@ -801,38 +788,40 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (raw_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (raw_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - /* - * Try to read some more data. This will certainly reset - * raw_buf_index to zero, and raw_buf_ptr must go with it. - */ - if (!CopyLoadRawBuf(cstate)) - hit_eof = true; - raw_buf_ptr = 0; - copy_buf_len = cstate->raw_buf_len; + /* + * Try to read some more data. This will certainly reset + * raw_buf_index to zero, and raw_buf_ptr must go with it. + */ + if (!CopyLoadRawBuf(cstate, COPY_READ_LINE_LOOKAHEAD)) + hit_eof = true; + raw_buf_ptr = 0; + copy_buf_len = cstate->raw_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (copy_buf_len <= 0) + if (copy_buf_len - raw_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -841,20 +830,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -888,14 +863,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of raw_buf. + * Look at the next character. If we're at EOF, c2 will wind + * up as '\0' because of the guaranteed pad of raw_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_raw_buf[raw_buf_ptr]; if (c == '\n') @@ -961,7 +931,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -976,16 +945,9 @@ CopyReadLineText(CopyFromState cstate) { raw_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 == '\n') @@ -1008,9 +970,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_raw_buf[raw_buf_ptr++]; if (c2 != '\r' && c2 != '\n') @@ -1095,7 +1055,6 @@ not_end_of_copy: mblen_str[0] = c; mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; } diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 705f5b615be..c088c4facdb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -70,8 +70,7 @@ typedef struct CopyFromStateData CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_NEW_FE */ - bool reached_eof; /* true if we read to end of copy data (not - * all copy_src types maintain this) */ + bool reached_eof; /* true if we read to end of copy data */ EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ -- 2.30.1 --------------5F54C02640018A10FACC0C96-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v3 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-04-01 20:22 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-04-01 20:22 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). Reviewed-by: John Naylor Discussion: https://www.postgresql.org/message-id/89627a2a-c123-a8aa-267e-5d168feb73dd%40iki.fi --- src/backend/commands/copyfromparse.c | 111 +++++++++------------------ 1 file changed, 35 insertions(+), 76 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 0813424768..b52abc1520 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -90,21 +90,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - input_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -159,7 +144,7 @@ static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, - int minread, int maxread); + int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); @@ -230,9 +215,8 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* * CopyGetData reads data from the source (file or frontend) * - * We attempt to read at least minread, and at most maxread, bytes from - * the source. The actual number of bytes read is returned; if this is - * less than minread, EOF was detected. + * We attempt to read at maxread bytes from the source. The actual + * number of bytes read is returned; if this is 0, EOF was detected. * * Note: when copying from the frontend, we expect a proper EOF mark per * protocol; if the frontend simply drops the connection, we raise error. @@ -241,7 +225,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) * NB: no data conversion is applied here. */ static int -CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) +CopyGetData(CopyFromState cstate, void *databuf, int maxread) { int bytesread = 0; @@ -257,7 +241,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) cstate->raw_reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread == 0 && !cstate->raw_reached_eof) { int avail; @@ -321,7 +305,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) } break; case COPY_CALLBACK: - bytesread = cstate->data_source_cb(databuf, minread, maxread); + bytesread = cstate->data_source_cb(databuf, 1, maxread); + if (bytesread == 0) + cstate->raw_reached_eof = true; break; } @@ -605,7 +591,7 @@ CopyLoadRawBuf(CopyFromState cstate) /* Load more data */ inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, - 1, RAW_BUF_SIZE - cstate->raw_buf_len); + RAW_BUF_SIZE - cstate->raw_buf_len); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_len = nbytes; @@ -990,7 +976,7 @@ CopyReadLine(CopyFromState cstate) do { inbytes = CopyGetData(cstate, cstate->input_buf, - 1, INPUT_BUF_SIZE); + INPUT_BUF_SIZE); } while (inbytes > 0); cstate->input_buf_index = 0; cstate->input_buf_len = 0; @@ -1047,8 +1033,7 @@ CopyReadLineText(CopyFromState cstate) char *copy_input_buf; int input_buf_ptr; int copy_buf_len; - bool need_data = false; - bool hit_eof = false; + bool hit_eof; bool result = false; /* CSV variables */ @@ -1098,6 +1083,7 @@ CopyReadLineText(CopyFromState cstate) copy_input_buf = cstate->input_buf; input_buf_ptr = cstate->input_buf_index; copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; for (;;) { @@ -1105,35 +1091,37 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead max three bytes in one iteration of the loop + * (for the sequence \.<CR><NL>), so make sure we have at least four + * bytes in the buffer. Note that we always guarantee that there is + * one \0 in the buffer, after last valid byte; the lookaheads below + * rely on that. */ - if (input_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (input_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - CopyLoadInputBuf(cstate); - /* update our local variables */ - hit_eof = cstate->input_reached_eof; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + CopyLoadInputBuf(cstate); + /* update our local variables */ + hit_eof = cstate->input_reached_eof; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (INPUT_BUF_BYTES(cstate) <= 0) + if (copy_buf_len - input_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -1142,20 +1130,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -1189,14 +1163,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of input_buf. + * Look at the next character. If we're at EOF, c will wind + * up as '\0' because of the guaranteed pad of input_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_input_buf[input_buf_ptr]; if (c == '\n') @@ -1262,7 +1231,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -1277,16 +1245,9 @@ CopyReadLineText(CopyFromState cstate) { input_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 == '\n') @@ -1309,9 +1270,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 != '\r' && c2 != '\n') -- 2.30.2 --------------11E1AF537C24312AF77A7698-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* [PATCH v4 1/1] Simplify COPY FROM parsing by forcing lookahead. @ 2021-04-06 17:48 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: Heikki Linnakangas @ 2021-04-06 17:48 UTC (permalink / raw) Now that we don't support the old-style COPY protocol anymore, we can force four bytes of lookahead like was suggested in an existing comment, and simplify the loop in CopyReadLineText(). FIXME: This fails with sequence "\.<NL><some invalid bytes>". We should detect the end-of-copy marker \. and stop reading without complaining about the garbage after the end-of-copy marker. That doesn't work if we force 4 bytes of lookahead; the invalid byte sequence fits in the lookahead window, so we will try to convert it. Reviewed-by: John Naylor Discussion: https://www.postgresql.org/message-id/89627a2a-c123-a8aa-267e-5d168feb73dd%40iki.fi --- src/backend/commands/copyfromparse.c | 112 +++++++++------------------ 1 file changed, 36 insertions(+), 76 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 0813424768..6b140531ed 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -90,21 +90,6 @@ * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros. */ -/* - * This keeps the character read at the top of the loop in the buffer - * even if there is more than one read-ahead. - */ -#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \ -if (1) \ -{ \ - if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \ - { \ - input_buf_ptr = prev_raw_ptr; /* undo fetch */ \ - need_data = true; \ - continue; \ - } \ -} else ((void) 0) - /* This consumes the remainder of the buffer and breaks */ #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \ if (1) \ @@ -159,7 +144,7 @@ static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, - int minread, int maxread); + int maxread); static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); @@ -230,9 +215,8 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* * CopyGetData reads data from the source (file or frontend) * - * We attempt to read at least minread, and at most maxread, bytes from - * the source. The actual number of bytes read is returned; if this is - * less than minread, EOF was detected. + * We attempt to read at most maxread bytes from the source. The actual + * number of bytes read is returned; if this is 0, EOF was detected. * * Note: when copying from the frontend, we expect a proper EOF mark per * protocol; if the frontend simply drops the connection, we raise error. @@ -241,7 +225,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) * NB: no data conversion is applied here. */ static int -CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) +CopyGetData(CopyFromState cstate, void *databuf, int maxread) { int bytesread = 0; @@ -257,7 +241,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) cstate->raw_reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread == 0 && !cstate->raw_reached_eof) { int avail; @@ -321,7 +305,9 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) } break; case COPY_CALLBACK: - bytesread = cstate->data_source_cb(databuf, minread, maxread); + bytesread = cstate->data_source_cb(databuf, 1, maxread); + if (bytesread == 0) + cstate->raw_reached_eof = true; break; } @@ -605,7 +591,7 @@ CopyLoadRawBuf(CopyFromState cstate) /* Load more data */ inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, - 1, RAW_BUF_SIZE - cstate->raw_buf_len); + RAW_BUF_SIZE - cstate->raw_buf_len); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_len = nbytes; @@ -990,7 +976,7 @@ CopyReadLine(CopyFromState cstate) do { inbytes = CopyGetData(cstate, cstate->input_buf, - 1, INPUT_BUF_SIZE); + INPUT_BUF_SIZE); } while (inbytes > 0); cstate->input_buf_index = 0; cstate->input_buf_len = 0; @@ -1047,8 +1033,7 @@ CopyReadLineText(CopyFromState cstate) char *copy_input_buf; int input_buf_ptr; int copy_buf_len; - bool need_data = false; - bool hit_eof = false; + bool hit_eof; bool result = false; /* CSV variables */ @@ -1098,6 +1083,7 @@ CopyReadLineText(CopyFromState cstate) copy_input_buf = cstate->input_buf; input_buf_ptr = cstate->input_buf_index; copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; for (;;) { @@ -1105,35 +1091,38 @@ CopyReadLineText(CopyFromState cstate) char c; /* - * Load more data if needed. Ideally we would just force four bytes - * of read-ahead and avoid the many calls to - * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol - * does not allow us to read too far ahead or we might read into the - * next data, so we read-ahead only as far we know we can. One - * optimization would be to read-ahead four byte here if - * cstate->copy_src != COPY_OLD_FE, but it hardly seems worth it, - * considering the size of the buffer. + * Load more data if needed. + * + * We need to look ahead at most three bytes in one iteration of the + * loop (for the sequence \.<CR><NL>), so make sure we have at least + * four bytes in the buffer. Note that we always guarantee that there + * is one \0 in the buffer, after last valid byte; the lookaheads + * below rely on that. */ - if (input_buf_ptr >= copy_buf_len || need_data) +#define COPY_READ_LINE_LOOKAHEAD 4 + if (input_buf_ptr + COPY_READ_LINE_LOOKAHEAD >= copy_buf_len) { - REFILL_LINEBUF; + if (!hit_eof) + { + REFILL_LINEBUF; - CopyLoadInputBuf(cstate); - /* update our local variables */ - hit_eof = cstate->input_reached_eof; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + CopyLoadInputBuf(cstate); + /* update our local variables */ + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + hit_eof = cstate->input_reached_eof; + continue; + } /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (INPUT_BUF_BYTES(cstate) <= 0) + if (copy_buf_len - input_buf_ptr <= 0) { result = true; break; } - need_data = false; } /* OK to fetch a character */ @@ -1142,20 +1131,6 @@ CopyReadLineText(CopyFromState cstate) if (cstate->opts.csv_mode) { - /* - * If character is '\\' or '\r', we may need to look ahead below. - * Force fetch of the next character if we don't already have it. - * We need to do this before changing CSV state, in case one of - * these characters is also the quote or escape character. - * - * Note: old-protocol does not like forced prefetch, but it's OK - * here since we cannot validly be at EOF. - */ - if (c == '\\' || c == '\r') - { - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - } - /* * Dealing with quotes and escapes here is mildly tricky. If the * quote char is also the escape char, there's no problem - we @@ -1189,14 +1164,9 @@ CopyReadLineText(CopyFromState cstate) cstate->eol_type == EOL_CRNL) { /* - * If need more data, go back to loop top to load it. - * - * Note that if we are at EOF, c will wind up as '\0' because - * of the guaranteed pad of input_buf. + * Look at the next character. If we're at EOF, c will wind + * up as '\0' because of the guaranteed pad of input_buf. */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - - /* get next char */ c = copy_input_buf[input_buf_ptr]; if (c == '\n') @@ -1262,7 +1232,6 @@ CopyReadLineText(CopyFromState cstate) { char c2; - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); IF_NEED_REFILL_AND_EOF_BREAK(0); /* ----- @@ -1277,16 +1246,9 @@ CopyReadLineText(CopyFromState cstate) { input_buf_ptr++; /* consume the '.' */ - /* - * Note: if we loop back for more data here, it does not - * matter that the CSV state change checks are re-executed; we - * will come back here with no important state changed. - */ if (cstate->eol_type == EOL_CRNL) { - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 == '\n') @@ -1309,9 +1271,7 @@ CopyReadLineText(CopyFromState cstate) } } - /* Get the next character */ - IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); - /* if hit_eof, c2 will become '\0' */ + /* Get next character. If hit_eof, c2 will become '\0' */ c2 = copy_input_buf[input_buf_ptr++]; if (c2 != '\r' && c2 != '\n') -- 2.30.2 --------------1601CF8503AB8D78BE138DD6-- ^ permalink raw reply [nested|flat] 50+ messages in thread
* Re: Fix database creation during installchecks for ICU cluster @ 2023-01-17 11:47 vignesh C <[email protected]> 2023-01-31 17:51 ` Re: Fix database creation during installchecks for ICU cluster vignesh C <[email protected]> 0 siblings, 1 reply; 50+ messages in thread From: vignesh C @ 2023-01-17 11:47 UTC (permalink / raw) To: Nazir Bilal Yavuz <[email protected]>; +Cc: Marina Polyakova <[email protected]>; pgsql-hackers On Tue, 29 Nov 2022 at 20:24, Nazir Bilal Yavuz <[email protected]> wrote: > > Hi, > > Thanks for the patch! > > > On 10/29/22 12:54, Marina Polyakova wrote: > > > > 1) The ECPG tests fail because they use the SQL_ASCII encoding [2], > > the database template0 uses the ICU locale provider and SQL_ASCII is > > not supported by ICU: > > > > $ make -C src/interfaces/ecpg/ installcheck > > ... > > ============== creating database "ecpg1_regression" ============== > > ERROR: encoding "SQL_ASCII" is not supported with ICU provider > > ERROR: database "ecpg1_regression" does not exist > > command failed: "/home/marina/postgresql/master/my/inst/bin/psql" -X > > -c "CREATE DATABASE \"ecpg1_regression\" TEMPLATE=template0 > > ENCODING='SQL_ASCII'" -c "ALTER DATABASE \"ecpg1_regression\" SET > > lc_messages TO 'C';ALTER DATABASE \"ecpg1_regression\" SET lc_monetary > > TO 'C';ALTER DATABASE \"ecpg1_regression\" SET lc_numeric TO 'C';ALTER > > DATABASE \"ecpg1_regression\" SET lc_time TO 'C';ALTER DATABASE > > \"ecpg1_regression\" SET bytea_output TO 'hex';ALTER DATABASE > > \"ecpg1_regression\" SET timezone_abbreviations TO 'Default';" "postgres" > > > > I can confirm that same error happens on my end and your patch fixes the > issue. But, do ECPG tests really require SQL_ASCII encoding? I removed > ECPG tests' encoding line [1], rebuilt it and 'make -C > src/interfaces/ecpg/ installcheck' passed without applying your patch. > > > > > > 2) The option --no-locale in pg_regress is described as "use C locale" > > [3]. But in this case the created databases actually use the ICU > > locale provider with the ICU cluster locale from template0 (see > > diff_check_backend_used_provider.txt): > > > > $ make NO_LOCALE=1 installcheck > > > This works on my end without applying your patch. Commands I used are: > > $ ./configure --with-icu --prefix=$PWD/build_dir > $ make && make install && export PATH=$PWD/build_dir/bin:$PATH > $ initdb --locale-provider icu --icu-locale en-US -D data && pg_ctl -D > data -l logfile start > $ make NO_LOCALE=1 installcheck Hi Marina Polyakova, Since it is working without your patch, Is this patch required for any other scenarios? Regards, Vignesh ^ permalink raw reply [nested|flat] 50+ messages in thread
* Re: Fix database creation during installchecks for ICU cluster 2023-01-17 11:47 Re: Fix database creation during installchecks for ICU cluster vignesh C <[email protected]> @ 2023-01-31 17:51 ` vignesh C <[email protected]> 0 siblings, 0 replies; 50+ messages in thread From: vignesh C @ 2023-01-31 17:51 UTC (permalink / raw) To: Nazir Bilal Yavuz <[email protected]>; +Cc: Marina Polyakova <[email protected]>; pgsql-hackers On Tue, 17 Jan 2023 at 17:17, vignesh C <[email protected]> wrote: > > On Tue, 29 Nov 2022 at 20:24, Nazir Bilal Yavuz <[email protected]> wrote: > > > > Hi, > > > > Thanks for the patch! > > > > > > On 10/29/22 12:54, Marina Polyakova wrote: > > > > > > 1) The ECPG tests fail because they use the SQL_ASCII encoding [2], > > > the database template0 uses the ICU locale provider and SQL_ASCII is > > > not supported by ICU: > > > > > > $ make -C src/interfaces/ecpg/ installcheck > > > ... > > > ============== creating database "ecpg1_regression" ============== > > > ERROR: encoding "SQL_ASCII" is not supported with ICU provider > > > ERROR: database "ecpg1_regression" does not exist > > > command failed: "/home/marina/postgresql/master/my/inst/bin/psql" -X > > > -c "CREATE DATABASE \"ecpg1_regression\" TEMPLATE=template0 > > > ENCODING='SQL_ASCII'" -c "ALTER DATABASE \"ecpg1_regression\" SET > > > lc_messages TO 'C';ALTER DATABASE \"ecpg1_regression\" SET lc_monetary > > > TO 'C';ALTER DATABASE \"ecpg1_regression\" SET lc_numeric TO 'C';ALTER > > > DATABASE \"ecpg1_regression\" SET lc_time TO 'C';ALTER DATABASE > > > \"ecpg1_regression\" SET bytea_output TO 'hex';ALTER DATABASE > > > \"ecpg1_regression\" SET timezone_abbreviations TO 'Default';" "postgres" > > > > > > > I can confirm that same error happens on my end and your patch fixes the > > issue. But, do ECPG tests really require SQL_ASCII encoding? I removed > > ECPG tests' encoding line [1], rebuilt it and 'make -C > > src/interfaces/ecpg/ installcheck' passed without applying your patch. > > > > > > > > > > 2) The option --no-locale in pg_regress is described as "use C locale" > > > [3]. But in this case the created databases actually use the ICU > > > locale provider with the ICU cluster locale from template0 (see > > > diff_check_backend_used_provider.txt): > > > > > > $ make NO_LOCALE=1 installcheck > > > > > > This works on my end without applying your patch. Commands I used are: > > > > $ ./configure --with-icu --prefix=$PWD/build_dir > > $ make && make install && export PATH=$PWD/build_dir/bin:$PATH > > $ initdb --locale-provider icu --icu-locale en-US -D data && pg_ctl -D > > data -l logfile start > > $ make NO_LOCALE=1 installcheck > > Hi Marina Polyakova, > > Since it is working without your patch, Is this patch required for any > other scenarios? There has been no updates on this thread for some time, so this has been switched as Returned with Feedback. Feel free to open it in the next commitfest if you plan to continue on this. Regards, Vignesh ^ permalink raw reply [nested|flat] 50+ messages in thread
end of thread, other threads:[~2023-01-31 17:51 UTC | newest] Thread overview: 50+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-02-04 10:39 [PATCH 2/2] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-03-04 09:10 [PATCH v2 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-04-01 20:22 [PATCH v3 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2021-04-06 17:48 [PATCH v4 1/1] Simplify COPY FROM parsing by forcing lookahead. Heikki Linnakangas <[email protected]> 2023-01-17 11:47 Re: Fix database creation during installchecks for ICU cluster vignesh C <[email protected]> 2023-01-31 17:51 ` Re: Fix database creation during installchecks for ICU cluster vignesh C <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox