From 7acaeb3201ae4ae279bf8b25641bea7f8cb92cbe Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Wed, 4 Mar 2026 17:28:54 +0300 Subject: [PATCH v11] Speed up COPY FROM text/CSV parsing using SIMD This patch disables SIMD when SIMD encounters a special character which is neither EOF nor EOL. Author: Shinya Kato Author: Nazir Bilal Yavuz Reviewed-by: Kazar Ayoub Reviewed-by: Nathan Bossart Reviewed-by: Neil Conway Reviewed-by: Andrew Dunstan Reviewed-by: Manni Wood Reviewed-by: Mark Wong Discussion: https://postgr.es/m/CAOzEurSW8cNr6TPKsjrstnPfhf4QyQqB4tnPXGGe8N4e_v7Jig%40mail.gmail.com --- src/backend/commands/copyfrom.c | 4 + src/backend/commands/copyfromparse.c | 222 ++++++++++++++++++++++- src/include/commands/copyfrom_internal.h | 4 + 3 files changed, 223 insertions(+), 7 deletions(-) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 2f42f55e229..2aa52810ff1 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1747,6 +1747,10 @@ BeginCopyFrom(ParseState *pstate, cstate->cur_attval = NULL; cstate->relname_only = false; + /* Initialize SIMD */ + cstate->simd_enabled = true; + cstate->simd_failed_first_vector = false; + /* * Allocate buffers for the input pipeline. * diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index fbd13353efc..70e1a5a0410 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -72,6 +72,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "port/pg_bswap.h" +#include "port/simd.h" #include "utils/builtins.h" #include "utils/rel.h" @@ -158,6 +159,12 @@ static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromSta int *nfields, bool is_csv); +/* SIMD functions */ +#ifndef USE_NO_SIMD +static bool CopyReadLineTextSIMDHelper(CopyFromState cstate, bool is_csv, + bool *temp_hit_eof, int *temp_input_buf_ptr); +#endif + /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, @@ -1310,6 +1317,182 @@ CopyReadLine(CopyFromState cstate, bool is_csv) return result; } +#ifndef USE_NO_SIMD +/* + * Use SIMD instructions to efficiently scan the input buffer for special + * characters (e.g., newline, carriage return, quote, and escape). This is + * faster than byte-by-byte iteration, especially on large buffers. + * + * Note that, SIMD may become slower when the input contains many special + * characters. To avoid this regression, we disable SIMD for the rest of the + * input once we encounter a special character which is neither EOF nor EOL. + * Also, SIMD is disabled when it encounters two consecutive short lines that + * SIMD can't create a full sized Vector, too. + */ +static bool +CopyReadLineTextSIMDHelper(CopyFromState cstate, bool is_csv, bool *temp_hit_eof, int *temp_input_buf_ptr) +{ + char quotec = '\0'; + char escapec = '\0'; + char *copy_input_buf; + int input_buf_ptr; + int copy_buf_len; + bool result = false; + bool unique_escapec = false; + bool first_vector = true; + Vector8 nl = vector8_broadcast('\n'); + Vector8 cr = vector8_broadcast('\r'); + Vector8 bs = vector8_broadcast('\\'); + Vector8 quote = vector8_broadcast(0); + Vector8 escape = vector8_broadcast(0); + + if (is_csv) + { + quotec = cstate->opts.quote[0]; + escapec = cstate->opts.escape[0]; + + quote = vector8_broadcast(quotec); + if (quotec != escapec) + { + unique_escapec = true; + escape = vector8_broadcast(escapec); + } + } + + /* For a little extra speed we copy these into local variables */ + copy_input_buf = cstate->input_buf; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + while (true) + { + /* Load more data if needed */ + if (sizeof(Vector8) >= copy_buf_len - input_buf_ptr) + { + REFILL_LINEBUF; + + CopyLoadInputBuf(cstate); + /* update our local variables */ + *temp_hit_eof = cstate->input_reached_eof; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + /* + * If we are completely out of data, break out of the loop, + * reporting EOF. + */ + if (INPUT_BUF_BYTES(cstate) <= 0) + { + result = true; + break; + } + } + + if (copy_buf_len - input_buf_ptr > sizeof(Vector8)) + { + Vector8 chunk; + Vector8 match = vector8_broadcast(0); + + /* Load a chunk of data into a vector register */ + vector8_load(&chunk, (const uint8 *) ©_input_buf[input_buf_ptr]); + + if (is_csv) + { + match = vector8_or(vector8_eq(chunk, nl), vector8_eq(chunk, cr)); + match = vector8_or(match, vector8_eq(chunk, quote)); + if (unique_escapec) + match = vector8_or(match, vector8_eq(chunk, escape)); + } + else + { + match = vector8_or(vector8_eq(chunk, nl), vector8_eq(chunk, cr)); + match = vector8_or(match, vector8_eq(chunk, bs)); + } + + /* Check if we found any special characters */ + if (vector8_is_highbit_set(match)) + { + /* + * Found a special character. Advance up to that point and let + * the scalar code handle it. + */ + uint32 mask; + int advance; + char c1, + c2; + bool simd_hit_eol, + simd_hit_eof; + + mask = vector8_highbit_mask(match); + advance = pg_rightmost_one_pos32(mask); + + input_buf_ptr += advance; + c1 = copy_input_buf[input_buf_ptr]; + + /* + * Since we stopped within the chunk and ((copy_buf_len - + * input_buf_ptr) > sizeof(Vector8)) is true, + * copy_input_buf[input_buf_ptr + 1] is guaranteed to be + * readable. + */ + c2 = copy_input_buf[input_buf_ptr + 1]; + + simd_hit_eof = (c1 == '\\' && c2 == '.' && !is_csv); + simd_hit_eol = (c1 == '\r' || c1 == '\n'); + + /* + * Do not disable SIMD when we hit EOL or EOF characters. In + * practice, it does not matter for EOF because parsing ends + * there, but we keep the behavior consistent. + */ + if (!(simd_hit_eof || simd_hit_eol)) + cstate->simd_enabled = false; + + /* + * We encountered a EOL or EOF on the first vector. This means + * lines are not long enough to skip fully sized vector. If + * this happens two times consecutively, then disable the + * SIMD. + */ + if (first_vector) + { + if (cstate->simd_failed_first_vector) + cstate->simd_enabled = false; + + cstate->simd_failed_first_vector = true; + } + + break; + } + else + { + /* No special characters found, so skip the entire chunk */ + input_buf_ptr += sizeof(Vector8); + first_vector = false; + } + } + + /* + * Although we refill linebuf, there is not enough character to fill + * full sized vector. This doesn't mean that we encountered a line + * that is not enough to fill a full sized vector. + * + * Scalar code will handle the rest for this line. Then, SIMD will + * continue from the next line. + */ + else + { + first_vector = false; + break; + } + } + + cstate->simd_failed_first_vector = first_vector; + *temp_input_buf_ptr = input_buf_ptr; + return result; +} +#endif + /* * CopyReadLineText - inner loop of CopyReadLine for text mode */ @@ -1338,6 +1521,38 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) escapec = '\0'; } + /* input_buf_ptr will be used in the SIMD Helper function */ + input_buf_ptr = cstate->input_buf_index; + +#ifndef USE_NO_SIMD + /* First try to run SIMD, then continue with the scalar path */ + if (cstate->simd_enabled) + { + int temp_input_buf_ptr = input_buf_ptr; + bool temp_hit_eof = false; + + result = CopyReadLineTextSIMDHelper(cstate, is_csv, &temp_hit_eof, + &temp_input_buf_ptr); + input_buf_ptr = temp_input_buf_ptr; + hit_eof = temp_hit_eof; + + /* Short exit from SIMD */ + if (result) + { + /* + * Transfer any still-uncopied data to line_buf. + */ + REFILL_LINEBUF; + + return result; + } + } +#endif + + /* For a little extra speed we copy these into local variables */ + copy_input_buf = cstate->input_buf; + copy_buf_len = cstate->input_buf_len; + /* * The objective of this loop is to transfer the entire next input line * into line_buf. Hence, we only care for detecting newlines (\r and/or @@ -1359,14 +1574,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) * character to examine; any characters from input_buf_index to * input_buf_ptr have been determined to be part of the line, but not yet * transferred to line_buf. - * - * For a little extra speed within the loop, we copy input_buf and - * input_buf_len into local variables. */ - copy_input_buf = cstate->input_buf; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; - for (;;) { int prev_raw_ptr; diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index f892c343157..4a748df8ac8 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -89,6 +89,10 @@ typedef struct CopyFromStateData const char *cur_attval; /* current att value for error messages */ bool relname_only; /* don't output line number, att, etc. */ + /* SIMD variables */ + bool simd_enabled; + bool simd_failed_first_vector; + /* * Working state */ -- 2.47.3