From: Alon Goldshuv <agoldshuv@greenplum.com>
To: pgsql-patches@postgresql.org
Subject: Re: COPY FROM performance improvements
Date: Thu, 14 Jul 2005 17:22:18 -0700
Message-ID: <BEFC4FCB.7017%agoldshuv@greenplum.com> (raw)
In-Reply-To: <BEE623DE.7F0A%llonergan@greenplum.com>
I revisited my patch and removed the code duplications that were there, and
added support for CSV with buffered input, so CSV now runs faster too
(although it is not as optimized as the TEXT format parsing). So now
TEXT,CSV and BINARY are all parsed in CopyFrom(), like in the original file.
Patch attached.
Greetings,
Alon.
Attachments:copy_parse_improvements_V15.patch (application/octet-stream, 78.0K) download
Index: src/backend/commands/copy.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/commands/copy.c,v
retrieving revision 1.247
diff -c -r1.247 copy.c
*** src/backend/commands/copy.c 10 Jul 2005 21:13:58 -0000 1.247
--- src/backend/commands/copy.c 15 Jul 2005 00:07:49 -0000
***************
*** 8,14 ****
*
*
* IDENTIFICATION
! * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.247 2005/07/10 21:13:58 tgl Exp $
*
*-------------------------------------------------------------------------
*/
--- 8,14 ----
*
*
* IDENTIFICATION
! * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.246 2005/06/28 05:08:53 tgl Exp $
*
*-------------------------------------------------------------------------
*/
***************
*** 50,55 ****
--- 50,56 ----
#define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
#define OCTVALUE(c) ((c) - '0')
+ #define COPY_BUF_SIZE 65536
/*
* Represents the different source/dest cases we need to worry about at
***************
*** 63,78 ****
} CopyDest;
/*
- * State indicator showing what stopped CopyReadAttribute()- */- typedef enum CopyReadResult- {- NORMAL_ATTR,- END_OF_LINE,- UNTERMINATED_FIELD- } CopyReadResult;- - /*
* Represents the end-of-line terminator type of the input
*/
typedef enum EolType
--- 64,69 ----
***************
*** 97,102 ****
--- 88,96 ----
static EolType eol_type; /* EOL type of input */
static int client_encoding; /* remote side's character encoding */
static int server_encoding; /* local encoding */
+ static char eol_ch[2]; /* The byte values of the 1 or 2 eol bytes */+ static bool client_encoding_only; /* true if client encoding is a non+ * supported server encoding */
/* these are just for error messages, see copy_in_error_callback */
static bool copy_binary; /* is it a binary copy? */
***************
*** 104,126 ****
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attribute_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attribute_buf's data buffer!
*/
! static StringInfoData attribute_buf;
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attribute_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
--- 98,141 ----
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
+ /*+ * Static variables for buffered input parsing + */+ static char input_buf[COPY_BUF_SIZE + 1]; /* extra byte for '\0' */+ static int buffer_index; /* input buffer index */+ static bool end_marker;+ static char *begloc;+ static char *endloc;+ static bool buf_done; /* finished processing the current buffer */+ static bool line_done; /* finished processing the whole line or+ * stopped in the middle */+ /* these are for CSV format */+ static bool in_quote;+ static bool last_was_esc;+ + /* these are for TEXT format */+ static bool esc_in_prevbuf; /* escape was last character of the data+ * input buffer */+ static bool cr_in_prevbuf; /* CR was last character of the data input+ * buffer */+
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attr_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attr_buf's data buffer!
*/
! static StringInfoData attr_buf;
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attr_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
***************
*** 137,155 ****
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLine(char * quote, char * escape);
! static char *CopyReadAttribute(const char *delim, const char *null_print,
! CopyReadResult *result, bool *isnull);
! static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
! char *quote, char *escape,
! CopyReadResult *result, bool *isnull);
static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! Oid typioparam, int32 typmod, bool *isnull);
static void CopyAttributeOut(char *string, char *delim);
static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
char *escape, bool force_quote);
static List *CopyGetAttnums(Relation rel, List *attnamelist);
static void limit_printout_length(StringInfo buf);
/* Internal communications functions */
static void SendCopyBegin(bool binary, int natts);
--- 152,176 ----
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLineText(size_t bytesread, char *escape);
! static bool CopyReadLineCSV(size_t bytesread, char *quote, char *escape);
! static void CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
! static void CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! char *escape, int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! Oid typioparam, bool *isnull);
! static char *CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
! bool *isnull);
static void CopyAttributeOut(char *string, char *delim);
static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
char *escape, bool force_quote);
static List *CopyGetAttnums(Relation rel, List *attnamelist);
static void limit_printout_length(StringInfo buf);
+ static bool DetectLineEnd(size_t bytesread, char *quote, char *escape);+
/* Internal communications functions */
static void SendCopyBegin(bool binary, int natts);
***************
*** 159,175 ****
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static void CopyGetData(void *databuf, int datasize);
! static int CopyGetChar(void);
#define CopyGetEof() (fe_eof)
- static int CopyPeekChar(void);- static void CopyDonePeek(int c, bool pickup);
static void CopySendInt32(int32 val);
static int32 CopyGetInt32(void);
static void CopySendInt16(int16 val);
static int16 CopyGetInt16(void);
/*
* Send copy start/stop messages for frontend copies. These have changed
--- 180,197 ----
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static int CopyGetData(void *databuf, int datasize);
#define CopyGetEof() (fe_eof)
static void CopySendInt32(int32 val);
static int32 CopyGetInt32(void);
static void CopySendInt16(int16 val);
static int16 CopyGetInt16(void);
+ /* byte scaning utils */+ static char *scanTextLine(const char *s, char c, size_t len);+ static char *scanCSVLine(const char *s, char c1, char c2, char c3, size_t len);+ static char *scanTextAttr(const char *s, char c1, char c2, size_t len);
/*
* Send copy start/stop messages for frontend copies. These have changed
***************
*** 382,395 ****
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
*/
! static void
CopyGetData(void *databuf, int datasize)
{
switch (copy_dest)
{
case COPY_FILE:
! fread(databuf, datasize, 1, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
--- 404,422 ----
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
+ *+ * Returns: the number of bytes that were successfully read+ * into the data buffer.
*/
! static int
CopyGetData(void *databuf, int datasize)
{
+ size_t bytesread = 0;+
switch (copy_dest)
{
case COPY_FILE:
! bytesread = fread(databuf, 1, datasize, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
***************
*** 401,406 ****
--- 428,435 ----
(errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("unexpected EOF on client connection")));
}
+ bytesread += datasize; /* update the count of bytes that were+ * read so far */
break;
case COPY_NEW_FE:
while (datasize > 0 && !fe_eof)
***************
*** 429,435 ****
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
--- 458,464 ----
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return bytesread;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
***************
*** 459,598 ****
avail = datasize;
pq_copymsgbytes(copy_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
datasize -= avail;
}
break;
}
- }- - static int- CopyGetChar(void)- {- int ch;- - switch (copy_dest)- {- case COPY_FILE:- ch = getc(copy_file);- break;- case COPY_OLD_FE:- ch = pq_getbyte();- if (ch == EOF)- {- /* Only a \. terminator is legal EOF in old protocol */- ereport(ERROR,- (errcode(ERRCODE_CONNECTION_FAILURE),- errmsg("unexpected EOF on client connection")));- }- break;- case COPY_NEW_FE:- {- unsigned char cc;- - CopyGetData(&cc, 1);- if (fe_eof)- ch = EOF;- else- ch = cc;- break;- }- default:- ch = EOF;- break;- }- if (ch == EOF)- fe_eof = true;- return ch;- }- - /*- * CopyPeekChar reads a byte in "peekable" mode.- *- * after each call to CopyPeekChar, a call to CopyDonePeek _must_- * follow, unless EOF was returned.- *- * CopyDonePeek will either take the peeked char off the stream- * (if pickup is true) or leave it on the stream (if pickup is false).- */- static int- CopyPeekChar(void)- {- int ch;- - switch (copy_dest)- {- case COPY_FILE:- ch = getc(copy_file);- break;- case COPY_OLD_FE:- ch = pq_peekbyte();- if (ch == EOF)- {- /* Only a \. terminator is legal EOF in old protocol */- ereport(ERROR,- (errcode(ERRCODE_CONNECTION_FAILURE),- errmsg("unexpected EOF on client connection")));- }- break;- case COPY_NEW_FE:- {- unsigned char cc;- - CopyGetData(&cc, 1);- if (fe_eof)- ch = EOF;- else- ch = cc;- break;- }- default:- ch = EOF;- break;- }- if (ch == EOF)- fe_eof = true;- return ch;- }- - static void- CopyDonePeek(int c, bool pickup)- {- if (fe_eof)- return; /* can't unget an EOF */- switch (copy_dest)- {- case COPY_FILE:- if (!pickup)- {- /* We don't want to pick it up - so put it back in there */- ungetc(c, copy_file);- }- /* If we wanted to pick it up, it's already done */- break;- case COPY_OLD_FE:- if (pickup)- {- /* We want to pick it up */- (void) pq_getbyte();- }
! /*
! * If we didn't want to pick it up, just leave it where it
! * sits
! */
! break;
! case COPY_NEW_FE:
! if (!pickup)
! {
! /* We don't want to pick it up - so put it back in there */
! copy_msgbuf->cursor--;
! }
! /* If we wanted to pick it up, it's already done */
! break;
! }
}
-
/*
* These functions do apply some data conversion
*/
--- 488,503 ----
avail = datasize;
pq_copymsgbytes(copy_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
+ bytesread += avail; /* update the count of bytes that were+ * read so far */
datasize -= avail;
}
break;
}
! return bytesread;
}
/*
* These functions do apply some data conversion
*/
***************
*** 968,980 ****
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attribute_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
--- 873,905 ----
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attr_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
+ /*+ * check if the client encoding is one of the 5 encodings+ * that are not supported as a server encodings.+ */+ switch (client_encoding)+ {+ case PG_SJIS:+ case PG_BIG5:+ case PG_GBK:+ case PG_UHC:+ case PG_GB18030:+ client_encoding_only = true;+ break;+ default:+ client_encoding_only = false;+ }+ + if(!csv_mode)+ escape = "\\"; /* default for text format */+
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
***************
*** 1105,1113 ****
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attribute_buf.data);
pfree(line_buf.data);
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
--- 1030,1039 ----
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attr_buf.data);
pfree(line_buf.data);
+
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
***************
*** 1345,1351 ****
{
bytea *outputbytes;
! outputbytes = DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
--- 1271,1278 ----
{
bytea *outputbytes;
! outputbytes =
! DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
***************
*** 1395,1404 ****
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attribute_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attribute_buf.data);
}
else
{
--- 1322,1331 ----
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attr_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attr_buf.data);
}
else
{
***************
*** 1406,1411 ****
--- 1333,1340 ----
if (line_buf_converted ||
client_encoding == server_encoding)
{
+ /* Strip off the newline */+ *(line_buf.data + line_buf.len - 1) = '\0';
limit_printout_length(&line_buf);
errcontext("COPY %s, line %d: \"%s\"",
copy_relname, copy_lineno,
***************
*** 1485,1491 ****
Oid in_func_oid;
Datum *values;
char *nulls;
- bool done = false;
bool isnull;
ResultRelInfo *resultRelInfo;
EState *estate = CreateExecutorState(); /* for ExecConstraints() */
--- 1414,1419 ----
***************
*** 1496,1501 ****
--- 1424,1434 ----
ExprContext *econtext; /* used for ExecEvalExpr for default atts */
MemoryContext oldcontext = CurrentMemoryContext;
ErrorContextCallback errcontext;
+ int *attr_offsets;+ int null_print_len; /* length of null print */+ bool no_more_data;+ ListCell *cur;+
tupDesc = RelationGetDescr(rel);
attr = tupDesc->attrs;
***************
*** 1607,1613 ****
}
}
! /* Prepare to catch AFTER triggers. */
AfterTriggerBeginQuery();
/*
--- 1540,1548 ----
}
}
! /*
! * Prepare to catch AFTER triggers.
! */
AfterTriggerBeginQuery();
/*
***************
*** 1671,1676 ****
--- 1606,1612 ----
values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
nulls = (char *) palloc(num_phys_attrs * sizeof(char));
+ attr_offsets = (int *) palloc(num_phys_attrs * sizeof(int));
/* Make room for a PARAM_EXEC value for domain constraint checks */
if (hasConstraints)
***************
*** 1691,1712 ****
errcontext.previous = error_context_stack;
error_context_stack = &errcontext;
! /* on input just throw the header line away */
! if (header_line)
{
! copy_lineno++;
! done = CopyReadLine(quote, escape) ;
}
! while (!done)
{
bool skip_tuple;
Oid loaded_oid = InvalidOid;
CHECK_FOR_INTERRUPTS();
- copy_lineno++;-
/* Reset the per-tuple exprcontext */
ResetPerTupleExprContext(estate);
--- 1627,1681 ----
errcontext.previous = error_context_stack;
error_context_stack = &errcontext;
! /*
! * initialize buffered scan variables.
! */
! if(csv_mode)
! {
! in_quote = false;
! last_was_esc = false;
! }
!
! null_print_len = strlen(null_print);
!
! /* Set up data buffer to hold a chunk of data */
! MemSet(input_buf, ' ', COPY_BUF_SIZE * sizeof(char));
! input_buf[COPY_BUF_SIZE] = '\0';
!
! no_more_data = false; /* no more input data to read from file or FE */
! line_done = true;
! buf_done = false;
!
! do
{
! size_t bytesread = 0;
!
! /* read a chunk of data into the buffer */
! if (!binary)
! {
! bytesread = CopyGetData(input_buf, COPY_BUF_SIZE);
! buf_done = false;
!
! /* set buffer pointers to beginning of the buffer */
! begloc = input_buf;
! buffer_index = 0;
}
! /*
! * continue if some bytes were read or if we didn't reach EOF. if we
! * both reached EOF _and_ no bytes were read, quit the loop we are
! * done
! */
! if (bytesread > 0 || !fe_eof || binary)
! {
!
! while (!buf_done)
{
bool skip_tuple;
Oid loaded_oid = InvalidOid;
CHECK_FOR_INTERRUPTS();
/* Reset the per-tuple exprcontext */
ResetPerTupleExprContext(estate);
***************
*** 1716,1746 ****
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
if (!binary)
{
- CopyReadResult result = NORMAL_ATTR;- char *string;- ListCell *cur;
/* Actually read the line into memory here */
! done = csv_mode ?
! CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
/*
! * EOF at start of line means we're done. If we see EOF after
! * some characters, we act as though it was newline followed
! * by EOF, ie, process the line and then exit loop on next
! * iteration.
*/
! if (done && line_buf.len == 0)
break;
if (file_has_oids)
{
/* can't be in CSV mode here */
! string = CopyReadAttribute(delim, null_print,
! &result, &isnull);
if (isnull)
ereport(ERROR,
--- 1685,1729 ----
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
+ /* reset attribute pointers */+ MemSet(attr_offsets, 0, num_phys_attrs * sizeof(int));
if (!binary)
{
/* Actually read the line into memory here */
! line_done = csv_mode ?
! CopyReadLineCSV(bytesread, quote, escape) :
! CopyReadLineText(bytesread, escape);
/*
! * if finished processing data line - increment line count.
! * Otherwise, if eof is not yet reached, we skip att parsing
! * and read more data. But if eof _was_ reached it means
! * that the original last data line is defective and
! * we want to catch that error later on.
*/
! if (line_done)
! copy_lineno++;
! else if (!fe_eof || end_marker )
break;
+ if (header_line)+ {+ line_buf.len = 0; /* we can reset line buffer now. */+ line_buf.data[0] = '\0';+ line_buf.cursor = 0;+ header_line = false;+ continue;+ }+
if (file_has_oids)
{
+ char *oid_string;+
/* can't be in CSV mode here */
! oid_string = CopyReadOidAttr(delim, null_print, null_print_len,
! &isnull);
if (isnull)
ereport(ERROR,
***************
*** 1750,1756 ****
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
--- 1733,1739 ----
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(oid_string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
***************
*** 1759,1792 ****
}
}
! /* Loop to read the user attributes on the line. */
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
! /*
! * If prior attr on this line was ended by newline,
! * complain.
! */
! if (result != NORMAL_ATTR)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("missing data for column \"%s\"",
! NameStr(attr[m]->attname))));
! if (csv_mode)
! {
! string = CopyReadAttributeCSV(delim, null_print, quote,
! escape, &result, &isnull);
! if (result == UNTERMINATED_FIELD)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unterminated CSV quoted field")));
! }
else
! string = CopyReadAttribute(delim, null_print,
! &result, &isnull);
if (csv_mode && isnull && force_notnull[m])
{
--- 1742,1770 ----
}
}
! /* parse all the attribute in the data line */
! if(csv_mode)
! CopyReadAttributesCSV(delim, null_print, quote, escape, null_print_len,
! nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
! else
! CopyReadAttributesText(delim, escape, null_print, null_print_len,
! nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
!
! /*
! * Loop to read the user attributes on the line.
! */
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
+ char *string;
! string = attr_buf.data + attr_offsets[m];
! if (nulls[m] == ' ')
! isnull = false;
else
! isnull = true;
if (csv_mode && isnull && force_notnull[m])
{
***************
*** 1806,1823 ****
copy_attname = NULL;
}
}
- - /*- * Complain if there are more fields on the input line.- *- * Special case: if we're reading a zero-column table, we won't- * yet have called CopyReadAttribute() at all; so no error if- * line is empty.- */- if (result == NORMAL_ATTR && line_buf.len != 0)- ereport(ERROR,- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),- errmsg("extra data after last expected column")));
}
else
{
--- 1784,1789 ----
***************
*** 1828,1834 ****
fld_count = CopyGetInt16();
if (CopyGetEof() || fld_count == -1)
{
! done = true;
break;
}
--- 1794,1801 ----
fld_count = CopyGetInt16();
if (CopyGetEof() || fld_count == -1)
{
! buf_done = true;
! no_more_data = true;
break;
}
***************
*** 1845,1851 ****
DatumGetObjectId(CopyReadBinaryAttribute(0,
&oid_in_function,
oid_typioparam,
- -1,
&isnull));
if (isnull || loaded_oid == InvalidOid)
ereport(ERROR,
--- 1812,1817 ----
***************
*** 1865,1871 ****
values[m] = CopyReadBinaryAttribute(i,
&in_functions[m],
typioparams[m],
- attr[m]->atttypmod,
&isnull);
nulls[m] = isnull ? 'n' : ' ';
copy_attname = NULL;
--- 1831,1836 ----
***************
*** 1885,1891 ****
nulls[defmap[i]] = ' ';
}
! /* Next apply any domain constraints */
if (hasConstraints)
{
ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
--- 1850,1858 ----
nulls[defmap[i]] = ' ';
}
! /*
! * Next apply any domain constraints
! */
if (hasConstraints)
{
ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
***************
*** 1912,1924 ****
}
}
! /* And now we can form the input tuple. */
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids && file_has_oids)
HeapTupleSetOid(tuple, loaded_oid);
! /* Triggers and stuff need to be invoked in query context. */
MemoryContextSwitchTo(oldcontext);
skip_tuple = false;
--- 1879,1895 ----
}
}
! /*
! * And now we can form the input tuple.
! */
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids && file_has_oids)
HeapTupleSetOid(tuple, loaded_oid);
! /*
! * Triggers and stuff need to be invoked in query context.
! */
MemoryContextSwitchTo(oldcontext);
skip_tuple = false;
***************
*** 1945,1955 ****
/* Place tuple in tuple slot */
ExecStoreTuple(tuple, slot, InvalidBuffer, false);
! /* Check the constraints of the tuple */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
! /* OK, store the tuple and create index entries for it */
simple_heap_insert(rel, tuple);
if (resultRelInfo->ri_NumIndices > 0)
--- 1916,1930 ----
/* Place tuple in tuple slot */
ExecStoreTuple(tuple, slot, InvalidBuffer, false);
! /*
! * Check the constraints of the tuple
! */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
! /*
! * OK, store the tuple and create index entries for it
! */
simple_heap_insert(rel, tuple);
if (resultRelInfo->ri_NumIndices > 0)
***************
*** 1958,1976 ****
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
}
! /* Done, clean up */
error_context_stack = errcontext.previous;
MemoryContextSwitchTo(oldcontext);
! /* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, resultRelInfo);
! /* Handle queued AFTER triggers */
AfterTriggerEndQuery(estate);
pfree(values);
pfree(nulls);
--- 1933,1970 ----
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
+ + line_buf.len = 0; /* we can reset line buffer now. */+ line_buf.data[0] = '\0';+ line_buf.cursor = 0;+ } /* end while(!buf_done) */+ } /* end if (bytesread > 0 || !fe_eof) */+ else+ /* no bytes read, end of data */+ {+ no_more_data = true;
}
+ } while (!no_more_data);
! /*
! * Done, clean up
! */
error_context_stack = errcontext.previous;
MemoryContextSwitchTo(oldcontext);
! /*
! * Execute AFTER STATEMENT insertion triggers
! */
ExecASInsertTriggers(estate, resultRelInfo);
! /*
! * Handle queued AFTER triggers
! */
AfterTriggerEndQuery(estate);
+ pfree(attr_offsets);+
pfree(values);
pfree(nulls);
***************
*** 1990,2262 ****
/*
! * Read the next input line and stash it in line_buf, with conversion to
! * server encoding.
! *
! * Result is true if read was terminated by EOF, false if terminated
! * by newline.
*/
static bool
! CopyReadLine(char * quote, char * escape)
{
! bool result;
! bool change_encoding = (client_encoding != server_encoding);
! int c;
! int mblen;
! int j;
! unsigned char s[2];
char *cvt;
! bool in_quote = false, last_was_esc = false, csv_mode = false;
! char quotec = '\0', escapec = '\0';
!
! if (quote)
! {
! csv_mode = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
!
!
! s[1] = 0;
!
! /* reset line_buf to empty */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
/* mark that encoding conversion hasn't occurred yet */
line_buf_converted = false;
! /* set default status */
! result = false;
/*
! * In this loop we only care for detecting newlines (\r and/or \n) and
! * the end-of-copy marker (\.).
! *
! * In Text mode, for backwards compatibility we allow
! * backslashes to escape newline characters. Backslashes other than
! * the end marker get put into the line_buf, since CopyReadAttribute
! * does its own escape processing.
! *
! * In CSV mode, CR and NL inside q quoted field are just part of the
! * data value and are put in line_buf. We keep just enough state
! * to know if we are currently in a quoted field or not.
! *
! * These four characters, and only these four, are assumed the same in
! * frontend and backend encodings.
! *
! * We do not assume that second and later bytes of a frontend
! * multibyte character couldn't look like ASCII characters.
*/
! for (;;)
{
! c = CopyGetChar();
! if (c == EOF)
{
! result = true;
! break;
}
! if (csv_mode)
{
/*
! * Dealing with quotes and escapes here is mildly tricky. If the
! * quote char is also the escape char, there's no problem - we
! * just use the char as a toggle. If they are different, we need
! * to ensure that we only take account of an escape inside a quoted
! * field and immediately preceding a quote char, and not the
! * second in a escape-escape sequence.
*/
! if (in_quote && c == escapec)
! last_was_esc = ! last_was_esc;
! if (c == quotec && ! last_was_esc)
! in_quote = ! in_quote;
! if (c != escapec)
! last_was_esc = false;
/*
! * updating the line count for embedded CR and/or LF chars is
! * necessarily a little fragile - this test is probably about
! * the best we can do.
*/
! if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n'))
! copy_lineno++;
! }
! if (!in_quote && c == '\r')
{
! if (eol_type == EOL_NL)
{
! if (! csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in CSV data"),
! errhint("Use quoted CSV field to represent carriage return.")));
}
! /* Check for \r\n on first line, _and_ handle \r\n. */
! if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
{
! int c2 = CopyPeekChar();
! if (c2 == '\n')
! {
! CopyDonePeek(c2, true); /* eat newline */
! eol_type = EOL_CRNL;
}
else
{
! /* found \r, but no \n */
if (eol_type == EOL_CRNL)
{
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in data"),
! errhint("Use quoted CSV field to represent carriage return.")));
!
}
/*
! * if we got here, it is the first line and we didn't
! * get \n, so put it back
*/
! CopyDonePeek(c2, false);
! eol_type = EOL_CR;
}
}
- break;
}
! if (!in_quote && c == '\n')
{
! if (eol_type == EOL_CR || eol_type == EOL_CRNL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted newline found in data"),
! errhint("Use quoted CSV field to represent newline.")));
!
}
- eol_type = EOL_NL;- break;- }
! if ((line_buf.len == 0 || !csv_mode) && c == '\\')
! {
! int c2;
!
! if (csv_mode)
! c2 = CopyPeekChar();
! else
! c2 = c = CopyGetChar();
! if (c2 == EOF)
{
! result = true;
! if (csv_mode)
! CopyDonePeek(c2, true);
break;
}
! if (c2 == '.')
! {
! if (csv_mode)
! CopyDonePeek(c2, true); /* allow keep calling GetChar() */
! if (eol_type == EOL_CRNL)
! {
! c = CopyGetChar();
! if (c == '\n')
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker does not match previous newline style")));
! if (c != '\r')
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker corrupt")));
}
- c = CopyGetChar();- if (c != '\r' && c != '\n')- ereport(ERROR,- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),- errmsg("end-of-copy marker corrupt")));- if ((eol_type == EOL_NL && c != '\n') ||- (eol_type == EOL_CRNL && c != '\n') ||- (eol_type == EOL_CR && c != '\r'))- ereport(ERROR,- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),- errmsg("end-of-copy marker does not match previous newline style")));
/*
! * In protocol version 3, we should ignore anything after
! * \. up to the protocol end of copy data. (XXX maybe
! * better not to treat \. as special?)
*/
! if (copy_dest == COPY_NEW_FE)
{
! while (c != EOF)
! c = CopyGetChar();
}
! result = true; /* report EOF */
break;
}
! if (csv_mode)
! CopyDonePeek(c2, false); /* not a dot, so put it back */
else
! /* not EOF mark, so emit \ and following char literally */
! appendStringInfoCharMacro(&line_buf, '\\');
}
! appendStringInfoCharMacro(&line_buf, c);
/*
! * When client encoding != server, must be careful to read the
! * extra bytes of a multibyte character exactly, since the
! * encoding might not ensure they don't look like ASCII. When the
! * encodings are the same, we need not do this, since no server
! * encoding we use has ASCII-like following bytes.
*/
! if (change_encoding)
! {
! s[0] = c;
! mblen = pg_encoding_mblen(client_encoding, s);
! for (j = 1; j < mblen; j++)
! {
! c = CopyGetChar();
! if (c == EOF)
{
! result = true;
break;
}
- appendStringInfoCharMacro(&line_buf, c);
}
! if (result)
! break; /* out of outer loop */
}
- } /* end of outer loop */
! /* Done reading the line. Convert it to server encoding. */
! if (change_encoding)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
--- 1984,2422 ----
/*
! * Finds the next TEXT line that is in the input buffer and loads
! * it into line_buf. Returns an indication if the line that was read
! * is complete (if an unescaped line-end was encountered). If we
! * reached the end of buffer before the whole line was written into the
! * line buffer then returns false.
*/
static bool
! CopyReadLineText(size_t bytesread, char *escape)
{
! int linesize;
! bool transcode = (client_encoding != server_encoding);
char *cvt;
! char escapec = '\0';
/* mark that encoding conversion hasn't occurred yet */
line_buf_converted = false;
! /*
! * set the escape char for text format ('\\' by default).
! */
! escapec = escape[0];
/*
! * Detect end of line type if not already detected.
*/
! if (eol_type == EOL_UNKNOWN)
{
! bool eol_detected = DetectLineEnd(bytesread, NULL, escape);
!
! if (!eol_detected)
{
! /* load entire input buffer into line buf, and quit */
! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! buf_done = true;
! return false;
! }
}
! /*
! * Special case: eol is CRNL, last byte of previous buffer was an
! * unescaped CR and 1st byte of current buffer is NL. We check for
! * that here.
! */
! if (eol_type == EOL_CRNL)
! {
! /* if we started scanning from the 1st byte of the buffer */
! if (begloc == input_buf)
! {
! /* and had a CR in last byte of prev buf */
! if (cr_in_prevbuf)
! {
! /*
! * if this 1st byte in buffer is 2nd byte of line end sequence
! * (linefeed)
! */
! if (*begloc == eol_ch[1])
{
/*
! * load that one linefeed byte and indicate we are done
! * with the data line
*/
+ appendBinaryStringInfo(&line_buf, begloc, 1);+ buffer_index++;+ begloc++;+ esc_in_prevbuf = false;+ cr_in_prevbuf = false;+ return true;+ }+ }
! cr_in_prevbuf = false;
! }
! }
/*
! * (we need a loop so that if eol_ch is found, but prev ch is backslash,
! * we can search for the next eol_ch)
*/
! while (true)
! {
! /* reached end of buffer */
! if ((endloc = scanTextLine(begloc, eol_ch[0], bytesread - buffer_index)) == NULL)
! {
! linesize = COPY_BUF_SIZE - (begloc - input_buf);
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! if (line_buf.len > 1)
{
! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
!
! if (*last_ch == escapec)
{
! esc_in_prevbuf = true;
!
! if (line_buf.len > 2)
! {
! last_ch--;
! if (*last_ch == escapec)
! esc_in_prevbuf = false;
}
! }
! else if (*last_ch == '\r')
{
! if (eol_type == EOL_CRNL)
! cr_in_prevbuf = true;
! }
! }
! line_done = false;
! buf_done = true;
! break;
}
else
+ /* found the 1st eol ch in input_buf. */
{
! bool eol_found = true;
! bool eol_escaped = true;
!
! /*
! * Load that piece of data (potentially a data line) into the line buffer,
! * and update the pointers for the next scan.
! */
! linesize = endloc - begloc + 1;
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! buffer_index += linesize;
! begloc = endloc + 1;
!
if (eol_type == EOL_CRNL)
{
! /* check if there is a '\n' after the '\r' */
! if (*(endloc + 1) == '\n')
! {
! /* this is a line end */
! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */
! buffer_index++;
! begloc++;
! }
else
! /* just a CR, not a line end */
! eol_found = false;
}
/*
! * in some cases, this end of line char happens to be the
! * last character in the buffer. we need to catch that.
*/
! if (buffer_index >= bytesread)
! buf_done = true;
!
! /*
! * Check if the 1st end of line ch is escaped.
! */
! if (endloc != input_buf) /* can we look 1 char back? */
! {
! if (*(endloc - 1) != escapec) /* prev char is not an escape */
! eol_escaped = false;
! else
! /* prev char is an escape */
! {
! if (endloc != (input_buf + 1)) /* can we look another
! * char back? */
! {
! /* it's a double escape char, so it's not an escape */
! if (*(endloc - 2) == escapec)
! eol_escaped = false;
! /* else it's a single escape char, so EOL is ascaped */
! }
! else
! {
! /* we need to check in the last buffer */
! if (esc_in_prevbuf)
! eol_escaped = false;
}
}
}
! else
! /*
! * this eol ch is first ch in buffer, check for escape in prev buf
! */
{
! if (!esc_in_prevbuf)
! eol_escaped = false;
}
! esc_in_prevbuf = false; /* reset variable */
! /*
! * if eol was found, and it isn't escaped, line is done
! */
! if ((eol_escaped == false) && eol_found)
{
! line_done = true;
break;
}
! else
! /* stay in the loop and process some more data. */
! line_done = false;
!
! } /* end of found eol_ch */
! }
! /*
! * Done reading the line. Convert it to server encoding.
! */
! if (transcode)
! {
! cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
! line_buf.len);
! if (cvt != line_buf.data)
! {
! /* transfer converted data back to line_buf */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
! appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
! }
! }
!
! /* indicate that conversion had occured */
! line_buf_converted = true;
!
! /*
! * check if this line is an end marker -- "\."
! */
! end_marker = false;
!
! switch (eol_type)
! {
! case EOL_NL:
! if (!strcmp(line_buf.data, "\\.\n"))
! end_marker = true;
! break;
! case EOL_CR:
! if (!strcmp(line_buf.data, "\\.\r"))
! end_marker = true;
! break;
! case EOL_CRNL:
! if (!strcmp(line_buf.data, "\\.\r\n"))
! end_marker = true;
! break;
! case EOL_UNKNOWN:
! break;
! }
!
! if (end_marker)
! {
! fe_eof = true;
! /* we don't want to process a \. as data line, want to quit. */
! line_done = false;
! buf_done = true;
! }
!
! return line_done;
! }
!
! /*
! * Finds the next CSV line that is in the input buffer and loads
! * it into line_buf. Returns an indication if the line that was read
! * is complete (if an unescaped line-end was encountered). If we
! * reached the end of buffer before the whole line was written into the
! * line buffer then returns false.
! */
! static bool
! CopyReadLineCSV(size_t bytesread, char *quote, char *escape)
! {
! int linesize;
! char *cvt;
! char quotec = '\0',
! escapec = '\0';
! bool transcode = (client_encoding != server_encoding);
!
! /* mark that encoding conversion hasn't occurred yet */
! line_buf_converted = false;
!
! escapec = escape[0];
! quotec = quote[0];
!
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
!
! /*
! * Detect end of line type if not already detected.
! */
! if (eol_type == EOL_UNKNOWN)
! {
! bool eol_detected = DetectLineEnd(bytesread, quote, escape);
!
! if (!eol_detected)
! {
! /* load entire input buffer into line buf, and quit */
! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! line_done = false;
! buf_done = true;
!
! return line_done;
! }
! }
!
! /*
! * Special case: eol is CRNL, last byte of previous buffer was an
! * unescaped CR and 1st byte of current buffer is NL. We check for
! * that here.
! */
! if (eol_type == EOL_CRNL)
! {
! /* if we started scanning from the 1st byte of the buffer */
! if (begloc == input_buf)
! {
! /* and had a CR in last byte of prev buf */
! if (cr_in_prevbuf)
! {
! /*
! * if this 1st byte in buffer is 2nd byte of line end sequence
! * (linefeed)
! */
! if (*begloc == eol_ch[1])
! {
! /*
! * load that one linefeed byte and indicate we are done
! * with the data line
! */
! appendBinaryStringInfo(&line_buf, begloc, 1);
! buffer_index++;
! begloc++;
!
! line_done = true;
! esc_in_prevbuf = false;
! cr_in_prevbuf = false;
!
! return line_done;
! }
! }
!
! cr_in_prevbuf = false;
! }
}
/*
! * (we need a loop so that if eol_ch is found, but we are in quotes,
! * we can search for the next eol_ch)
*/
! while (true)
! {
! /* reached end of buffer */
! if ((endloc = scanCSVLine(begloc, eol_ch[0], escapec, quotec, bytesread - buffer_index)) == NULL)
{
! linesize = COPY_BUF_SIZE - (begloc - input_buf);
! appendBinaryStringInfo(&line_buf, begloc, linesize);
!
! if (last_was_esc)
! esc_in_prevbuf = true;
!
! if (line_buf.len > 1)
! {
! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
!
! if (*last_ch == '\r')
! {
! if (eol_type == EOL_CRNL)
! cr_in_prevbuf = true;
}
! }
!
! line_done = false;
! buf_done = true;
break;
}
+ else+ /* found 1st eol char in input_buf. */+ {+ bool eol_found = true;
! /*
! * Load that piece of data (potentially a data line) into the line buffer,
! * and update the pointers for the next scan.
! */
! linesize = endloc - begloc + 1;
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! buffer_index += linesize;
! begloc = endloc + 1;
!
! /* end of line only if not in quotes */
! if(in_quote)
! {
! line_done = false;
! /* buf done, but still in quote */
! if (buffer_index >= bytesread)
! {
! buf_done = true;
! break;
! }
! }
else
! {
! /* if dos eol, check for '\n' after the '\r' */
! if (eol_type == EOL_CRNL)
! {
! if (*(endloc + 1) == '\n')
! {
! /* this is a line end */
! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */
! buffer_index++;
! begloc++;
! }
! else
! /* just a CR, not a line end */
! eol_found = false;
}
! /*
! * in some cases, this end of line char happens to be the
! * last character in the buffer. we need to catch that.
! */
! if (buffer_index >= bytesread)
! buf_done = true;
/*
! * if eol was found line is done
*/
! if (eol_found)
{
! line_done = true;
break;
}
}
! } /* end of found eol_ch */
}
! /*
! * Done reading the line. Convert it to server encoding.
! */
! if (transcode)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
***************
*** 2265,2285 ****
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* Now it's safe to use the buffer in error messages */
line_buf_converted = true;
! return result;
}
/*
* Return decimal value for a hexadecimal digit
*/
static
! int GetDecimalFromHex(char hex)
{
if (isdigit(hex))
return hex - '0';
--- 2425,2478 ----
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
+ line_buf.cursor = 0;
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* indicate that conversion had occured */
line_buf_converted = true;
! /*
! * check if this line is an end marker -- "\."
! */
! end_marker = false;
!
! switch (eol_type)
! {
! case EOL_NL:
! if (!strcmp(line_buf.data, "\\.\n"))
! end_marker = true;
! break;
! case EOL_CR:
! if (!strcmp(line_buf.data, "\\.\r"))
! end_marker = true;
! break;
! case EOL_CRNL:
! if (!strcmp(line_buf.data, "\\.\r\n"))
! end_marker = true;
! break;
! case EOL_UNKNOWN:
! break;
! }
!
! if (end_marker)
! {
! fe_eof = true;
! /* we don't want to process a \. as data line, want to quit. */
! line_done = false;
! buf_done = true;
! }
!
! return line_done;
}
/*
* Return decimal value for a hexadecimal digit
*/
static
! int
! GetDecimalFromHex(char hex)
{
if (isdigit(hex))
return hex - '0';
***************
*** 2287,2344 ****
return tolower(hex) - 'a' + 10;
}
! /*----------
! * Read the value of a single attribute, performing de-escaping as needed.
! *
! * delim is the column delimiter string (must be just one byte for now).
! * null_print is the null marker string. Note that this is compared to
! * the pre-de-escaped input string.
! *
! * *result is set to indicate what terminated the read:
! * NORMAL_ATTR: column delimiter
! * END_OF_LINE: end of line
! * In either case, the string read up to the terminator is returned.
! *
! * *isnull is set true or false depending on whether the input matched
! * the null marker. Note that the caller cannot check this since the
! * returned string will be the post-de-escaping equivalent, which may
! * look the same as some valid data string.
! *----------
*/
! static char *
! CopyReadAttribute(const char *delim, const char *null_print,
! CopyReadResult *result, bool *isnull)
{
char c;
! char delimc = delim[0];
! int start_cursor = line_buf.cursor;
! int end_cursor;
! int input_len;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
! /* set default status */
! *result = END_OF_LINE;
! for (;;)
{
! end_cursor = line_buf.cursor;
! if (line_buf.cursor >= line_buf.len)
! break;
! c = line_buf.data[line_buf.cursor++];
! if (c == delimc)
{
! *result = NORMAL_ATTR;
! break;
}
! if (c == '\\')
{
! if (line_buf.cursor >= line_buf.len)
! break;
! c = line_buf.data[line_buf.cursor++];
! switch (c)
{
case '0':
case '1':
--- 2480,2769 ----
return tolower(hex) - 'a' + 10;
}
! /*
! * Detected the eol type by looking at the first data row.
! * Possible eol types are NL, CR, or CRNL. If eol type was
! * detected, it is set and a boolean true is returned to
! * indicated detection was successful. If the first data row
! * is longer than the input buffer, we return false and will
! * try again in the next buffer.
*/
! static bool
! DetectLineEnd(size_t bytesread, char *quote, char *escape)
{
+ int index = 0;
char c;
! char quotec = '\0',
! escapec = '\0';
! bool csv = false;
! if (quote) /* CSV format */
! {
! csv = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
! while (index < COPY_BUF_SIZE)
! {
! c = input_buf[index];
! if(csv)
{
! if (in_quote && c == escapec)
! last_was_esc = !last_was_esc;
! if (c == quotec && !last_was_esc)
! in_quote = !in_quote;
! if (c != escapec)
! last_was_esc = false;
! }
!
! if (c == '\n')
{
! if(!csv || (csv && !in_quote))
! {
! eol_type = EOL_NL;
! eol_ch[0] = '\n';
! eol_ch[1] = '\0';
!
! in_quote = false;
! last_was_esc = false;
! return true;
! }
}
! if (c == '\r')
{
! if(!csv || (csv && !in_quote))
! {
! if (input_buf[index + 1] == '\n') /* always safe */
! {
! eol_type = EOL_CRNL;
! eol_ch[0] = '\r';
! eol_ch[1] = '\n';
! }
! else
! {
! eol_type = EOL_CR;
! eol_ch[0] = '\r';
! eol_ch[1] = '\0';
! }
!
! in_quote = false;
! last_was_esc = false;
! return true;
! }
! }
!
! index++;
! }
!
! return false;
! }
!
! /*
! * Read all TEXT attributes. Attributes are parsed from line_buf and
! * inserted (all at once) to attr_buf, while saving pointers to
! * each attribute's starting position.
! *
! * When this routine finishes execution both the nulls array and
! * the attr_offsets array are updated. The attr_offsets will include
! * the offset from the beginning of the attribute array of which
! * each attribute begins. If a specific attribute is not used for this
! * COPY command (ommitted from the column list), a value of 0 will be assigned.
! * For example: for table foo(a,b,c,d,e) and COPY foo(a,b,e)
! * attr_offsets may look something like this after this routine
! * returns: [0,20,0,0,55]. That means that column "a" value starts
! * at byte offset 0, "b" in 20 and "e" in 55, in attr_buf.
! *
! * In the attribute buffer (attr_buf) each attribute
! * is terminated with a '\0', and therefore by using the attr_offsets
! * array we could point to a beginning of an attribute and have it
! * behave as a C string, much like previously done in COPY.
! *
! * Another aspect to improving performance is reducing the frequency
! * of data load into buffers. The original COPY read attribute code
! * loaded a character at a time. In here we try to load a chunk of data
! * at a time. Usually a chunk will include a full data row
! * (unless we have an escaped delim). That effectively reduces the number of
! * loads by a factor of number of bytes per row. This improves performance
! * greatly, unfortunately it add more complexity to the code.
! *
! * Global participants in parsing logic:
! *
! * line_buf.cursor -- an offset from beginning of the line buffer
! * that indicates where we are about to begin the next scan. Note that
! * if we have WITH OIDS this cursor is already shifted past the first
! * OID attribute.
! *
! * attr_buf.cursor -- an offset from the beginning of the
! * attribute buffer that indicates where the current attribute begins.
! */
! static void
! CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
! {
! char delimc = delim[0]; /* delimiter character */
! char escapec = escape[0]; /* escape character */
! char *scan_start; /* pointer to line buffer for scan start. */
! char *scan_end; /* pointer to line buffer where char was found */
! int attr_pre_len; /* attr raw len, before processing escapes */
! int attr_post_len; /* current attr len after escaping */
! int m; /* attribute index being parsed */
! int bytes_remaining;/* num bytes remaining to be scanned in line buf */
! int chunk_start; /* offset to beginning of line chunk to load */
! int chunk_len; /* length of chunk of data to load to attr buf */
! int oct_val; /* byte value for octal escapes */
! int hex_val; /* byte value for hexadecimal escapes */
! char hexchar; /* char that appears after \x for hex values */
! int attnum; /* attribute number being parsed */
! ListCell *cur; /* cursor to attribute list used for this COPY */
! int attribute;
!
! /*
! * init variables for attribute scan
! */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
! /* cursor is now > 0 if we copy WITH OIDS */
! scan_start = line_buf.data + line_buf.cursor;
! cur = list_head(attnumlist);
! attnum = lfirst_int(cur);
! m = attnum - 1;
! chunk_start = line_buf.cursor;
! chunk_len = 0;
! attr_pre_len = 0;
! attr_post_len = 0;
!
! /*
! * Scan through the line buffer to read all attributes data
! */
! while (line_buf.cursor < line_buf.len)
! {
! bytes_remaining = line_buf.len - line_buf.cursor;
!
! if ((scan_end = scanTextAttr(scan_start, delimc, escapec, bytes_remaining))
! == NULL)
! {
! /* GOT TO END OF LINE BUFFER */
!
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* don't count eol char(s) in attr and chunk len calculation */
! if (eol_type == EOL_CRNL)
! {
! attr_pre_len += bytes_remaining - 2;
! chunk_len = line_buf.len - chunk_start - 2;
! }
! else
! {
! attr_pre_len += bytes_remaining - 1;
! chunk_len = line_buf.len - chunk_start - 1;
! }
!
! /* check if this is a NULL value or data value (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(line_buf.data + line_buf.len - attr_pre_len - 1, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! attr_offsets[m] = attr_buf.cursor;
!
!
! /* load the last chunk, the whole buffer in most cases */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! line_buf.cursor += attr_pre_len + 2; /* skip eol char and
! * '\0' to exit loop */
!
! if (lnext(cur) != NULL)
! {
! /*
! * For an empty data line, the previous COPY code will
! * fail it during the conversion stage. We can fail it here
! * already, but then we will fail the regression tests b/c
! * of a different error message. that's why we return so we
! * can get the same error message that regress expects. ahh...
! * this conditional is unnecessary and should be removed soon.
! */
! if (line_buf.len > 1)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("missing data for column \"%s\"",
! NameStr(attr[m + 1]->attname))));
! else
! return;
! }
! }
! else
! /* FOUND A DELIMITER OR ESCAPE */
! {
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! if (*scan_end == delimc) /* found a delimiter */
! {
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* (we don't include the delimiter ch in length) */
! attr_pre_len += scan_end - scan_start;
! /* (we don't include the delimiter ch in length) */
! attr_post_len += scan_end - scan_start;
!
! /* check if this is a null print or data (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(scan_end - attr_pre_len, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! /* set the pointer to next attribute position */
! attr_offsets[m] = attr_buf.cursor;
!
! /*
! * update buffer cursors to our current location, +1 to skip
! * the delimc
! */
! line_buf.cursor = scan_end - line_buf.data + 1;
! attr_buf.cursor += attr_post_len + 1;
!
! /* prepare scan for next attr */
! scan_start = line_buf.data + line_buf.cursor;
! cur = lnext(cur);
! attr_pre_len = 0;
! attr_post_len = 0;
! }
! else
! /* found an escape character */
! {
! char nextc = *(scan_end + 1);
! char newc;
! int skip = 2;
!
! chunk_len = (scan_end - line_buf.data) - chunk_start + 1;
!
! /* load a chunk of data */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! switch (nextc)
{
case '0':
case '1':
***************
*** 2348,2445 ****
case '5':
case '6':
case '7':
! /* handle \013 */
! {
! int val;
! val = OCTVALUE(c);
! if (line_buf.cursor < line_buf.len)
! {
! c = line_buf.data[line_buf.cursor];
! if (ISOCTAL(c))
! {
! line_buf.cursor++;
! val = (val << 3) + OCTVALUE(c);
! if (line_buf.cursor < line_buf.len)
{
! c = line_buf.data[line_buf.cursor];
! if (ISOCTAL(c))
{
! line_buf.cursor++;
! val = (val << 3) + OCTVALUE(c);
}
}
! }
! }
! c = val & 0377;
! }
break;
case 'x':
/* Handle \x3F */
! if (line_buf.cursor < line_buf.len)
! {
! char hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! int val = GetDecimalFromHex(hexchar);
! line_buf.cursor++;
! if (line_buf.cursor < line_buf.len)
! {
! hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! line_buf.cursor++;
! val = (val << 4) + GetDecimalFromHex(hexchar);
! }
! }
! c = val & 0xff;
}
}
break;
case 'b':
! c = '\b';
break;
case 'f':
! c = '\f';
break;
case 'n':
! c = '\n';
break;
case 'r':
! c = '\r';
break;
case 't':
! c = '\t';
break;
case 'v':
! c = '\v';
break;
/*
! * in all other cases, take the char after '\'
! * literally
*/
}
! }
! appendStringInfoCharMacro(&attribute_buf, c);
}
! /* check whether raw input matched null marker */
! input_len = end_cursor - start_cursor;
! if (input_len == strlen(null_print) &&
! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
*isnull = true;
! else
! *isnull = false;
! return attribute_buf.data;
! }
/*
! * Read the value of a single attribute in CSV mode,
* performing de-escaping as needed. Escaping does not follow the normal
* PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
*
--- 2773,2944 ----
case '5':
case '6':
case '7':
! oct_val = OCTVALUE(nextc);
! nextc = *(scan_end + 2);
! /*
! * (no need for out bad access check since line if
! * buffered)
! */
! if (ISOCTAL(nextc))
{
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
! nextc = *(scan_end + 3);
! if (ISOCTAL(nextc))
{
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
}
}
! newc = oct_val & 0377; /* the escaped byte value */
break;
case 'x':
/* Handle \x3F */
! hexchar = *(scan_end + 2);
if (isxdigit(hexchar))
{
! skip++;
! hex_val = GetDecimalFromHex(hexchar);
! hexchar = *(scan_end + 3);
if (isxdigit(hexchar))
{
! skip++;
! hex_val = (hex_val << 4) + GetDecimalFromHex(hexchar);
}
+ newc = hex_val & 0xff;
}
+ else /* "\x" with no hex value */+ newc = nextc;
break;
case 'b':
! newc = '\b';
break;
case 'f':
! newc = '\f';
break;
case 'n':
! newc = '\n';
break;
case 'r':
! newc = '\r';
break;
case 't':
! newc = '\t';
break;
case 'v':
! newc = '\v';
! break;
! default:
! if (nextc == delimc)
! newc = delimc;
! else if (nextc == escapec)
! newc = escapec;
! else
! /* no escape sequence, take next char literaly */
! newc = nextc;
break;
+ }+ + /* update to current length, add escape and escaped chars */+ attr_pre_len += scan_end - scan_start + 2;+ /* update to current length, escaped char */+ attr_post_len += scan_end - scan_start + 1;
/*
! * Need to get rid of the escape character. This is done by
! * loading the chunk up to including the escape character
! * into the attribute buffer. Then overwritting the backslash
! * with the escaped sequence or char, and continuing to scan
! * from *after* the char than is after the escape in line buf.
*/
+ *(attr_buf.data + attr_buf.len - 1) = newc;+ line_buf.cursor = scan_end - line_buf.data + skip;+ scan_start = scan_end + skip;+ chunk_start = line_buf.cursor;+ chunk_len = 0;
}
!
! } /* end delimiter/backslash */
!
! } /* end line buffer scan. */
!
! /*
! * Replace all delimiters with NULL for string termination.
! * NOTE: only delimiters (NOT necessarily all delimc) are replaced.
! * Example (delimc = '|'):
! * - Before: f 1 | f \| 2 | f 3
! * - After : f 1 \0 f | 2 \0 f 3
! */
! for (attribute = 1; attribute < num_phys_attrs; attribute++)
! {
! if (attr_offsets[attribute] != 0)
! *(attr_buf.data + attr_offsets[attribute] - 1) = '\0';
}
! }
!
!
! /*
! * Read a binary attribute
! */
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, Oid typioparam,
! bool *isnull)
! {
! int32 fld_size;
! Datum result;
!
! fld_size = CopyGetInt32();
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
! if (fld_size == -1)
! {
*isnull = true;
! return (Datum) 0;
! }
! if (fld_size < 0)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid field size")));
! /* reset attr_buf to empty, and load raw data in it */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! enlargeStringInfo(&attr_buf, fld_size);
!
! CopyGetData(attr_buf.data, fld_size);
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
+ attr_buf.len = fld_size;+ attr_buf.data[fld_size] = '\0';+ + /* Call the column type's binary input converter */+ result = FunctionCall2(flinfo,+ PointerGetDatum(&attr_buf),+ ObjectIdGetDatum(typioparam));+ + /* Trouble if it didn't eat the whole buffer */+ if (attr_buf.cursor != attr_buf.len)+ ereport(ERROR,+ (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),+ errmsg("incorrect binary data format")));+ + *isnull = false;+ return result;+ }
/*
! * Read all the attributes of the data line in CSV mode,
* performing de-escaping as needed. Escaping does not follow the normal
* PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
*
***************
*** 2448,2472 ****
*
* null_print is the null marker string. Note that this is compared to
* the pre-de-escaped input string (thus if it is quoted it is not a NULL).
- *- * *result is set to indicate what terminated the read:- * NORMAL_ATTR: column delimiter- * END_OF_LINE: end of line- * UNTERMINATED_FIELD no quote detected at end of a quoted field- *- * In any case, the string read up to the terminator (or end of file)- * is returned.- *- * *isnull is set true or false depending on whether the input matched- * the null marker. Note that the caller cannot check this since the- * returned string will be the post-de-escaping equivalent, which may- * look the same as some valid data string.
*----------
*/
!
! static char *
! CopyReadAttributeCSV(const char *delim, const char *null_print, char *quote,
! char *escape, CopyReadResult *result, bool *isnull)
{
char delimc = delim[0];
char quotec = quote[0];
--- 2947,2958 ----
*
* null_print is the null marker string. Note that this is compared to
* the pre-de-escaped input string (thus if it is quoted it is not a NULL).
*----------
*/
! static void
! CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! char *escape, int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
{
char delimc = delim[0];
char quotec = quote[0];
***************
*** 2477,2502 ****
int input_len;
bool in_quote = false;
bool saw_quote = false;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
- /* set default status */- *result = END_OF_LINE;
for (;;)
{
end_cursor = line_buf.cursor;
if (line_buf.cursor >= line_buf.len)
break;
c = line_buf.data[line_buf.cursor++];
/* unquoted field delimiter */
if (!in_quote && c == delimc)
{
! *result = NORMAL_ATTR;
! break;
}
/* start of quoted field (or part of field) */
--- 2963,3043 ----
int input_len;
bool in_quote = false;
bool saw_quote = false;
+ int attnum; /* attribute number being parsed */+ int m; /* attribute index being parsed */+ ListCell *cur; /* cursor to attribute list used for this COPY */
! /*
! * init variables for attribute scan
! */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! cur = list_head(attnumlist);
! attnum = lfirst_int(cur);
! m = attnum - 1;
! input_len = 0;
for (;;)
{
end_cursor = line_buf.cursor;
+ + /* finished processing attributes in line */
if (line_buf.cursor >= line_buf.len)
+ {+ /* check whether raw input matched null marker */+ input_len = end_cursor - start_cursor;+ if (!saw_quote && input_len == null_print_len &&+ strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)+ nulls[m] = 'n';+ else+ nulls[m] = ' ';+ + if (in_quote)+ ereport(ERROR,+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),+ errmsg("unterminated CSV quoted field")));+ + if (lnext(cur) != NULL)+ ereport(ERROR,+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),+ errmsg("missing data for column \"%s\"",+ NameStr(attr[m + 1]->attname))));
break;
+ }+
c = line_buf.data[line_buf.cursor++];
/* unquoted field delimiter */
if (!in_quote && c == delimc)
{
! /* check whether raw input matched null marker */
! input_len = end_cursor - start_cursor;
! if (!saw_quote && input_len == null_print_len &&
! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! /* terminate attr string with '\0' */
! appendStringInfoCharMacro(&attr_buf, '\0');
! attr_buf.cursor++;
!
! /* setup next attribute scan */
! cur = lnext(cur);
!
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! attnum = lfirst_int(cur);
! m = attnum - 1;
! attr_offsets[m] = attr_buf.cursor;
! start_cursor = line_buf.cursor;
! continue;
}
/* start of quoted field (or part of field) */
***************
*** 2520,2527 ****
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attribute_buf, nextc);
line_buf.cursor++;
continue;
}
}
--- 3061,3069 ----
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attr_buf, nextc);
line_buf.cursor++;
+ attr_buf.cursor++;
continue;
}
}
***************
*** 2537,2616 ****
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attribute_buf, c);
}
- if (in_quote)- *result = UNTERMINATED_FIELD;- - /* check whether raw input matched null marker */- input_len = end_cursor - start_cursor;- if (!saw_quote && input_len == strlen(null_print) &&- strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)- *isnull = true;- else- *isnull = false;- - return attribute_buf.data;
}
/*
! * Read a binary attribute
*/
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! Oid typioparam, int32 typmod,
bool *isnull)
{
! int32 fld_size;
! Datum result;
!
! fld_size = CopyGetInt32();
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
! if (fld_size == -1)
! {
! *isnull = true;
! return (Datum) 0;
}
! if (fld_size < 0)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid field size")));
! /* reset attribute_buf to empty, and load raw data in it */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
! attribute_buf.cursor = 0;
! enlargeStringInfo(&attribute_buf, fld_size);
- CopyGetData(attribute_buf.data, fld_size);- if (CopyGetEof())- ereport(ERROR,- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),- errmsg("unexpected EOF in COPY data")));
! attribute_buf.len = fld_size;
! attribute_buf.data[fld_size] = '\0';
! /* Call the column type's binary input converter */
! result = FunctionCall3(flinfo,
! PointerGetDatum(&attribute_buf),
! ObjectIdGetDatum(typioparam),
! Int32GetDatum(typmod));
- /* Trouble if it didn't eat the whole buffer */- if (attribute_buf.cursor != attribute_buf.len)- ereport(ERROR,- (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),- errmsg("incorrect binary data format")));- *isnull = false;- return result;- }
/*
* Send text representation of one attribute, with conversion and escaping
--- 3079,3145 ----
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attr_buf, c);
! attr_buf.cursor++;
}
}
/*
! * Read the first attribute. This is mainly used to maintain support
! * for an OID column. All the rest of the columns will be read at once with
! * CopyReadAttributesText.
*/
! static char *
! CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
bool *isnull)
{
! char delimc = delim[0];
! char *start_loc = line_buf.data + line_buf.cursor;
! char *end_loc;
! int attr_len = 0;
! int bytes_remaining;
!
! /* reset attribute buf to empty */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! /* # of bytes that were not yet processed in this line */
! bytes_remaining = line_buf.len - line_buf.cursor;
!
! /* got to end of line */
! if ((end_loc = scanTextLine(start_loc, delimc, bytes_remaining)) == NULL)
! {
! attr_len = bytes_remaining - 1; /* don't count '\n' in len calculation */
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 2; /* skip '\n' and '\0' */
}
! else
! /* found a delimiter */
! {
! /*
! * (we don't care if delim was preceded with a backslash, because it's
! * an invalid OID anyway)
! */
! attr_len = end_loc - start_loc; /* we don't include the delimiter ch */
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 1;
! }
! /* check whether raw input matched null marker */
! if (attr_len == null_print_len && strncmp(start_loc, null_print, attr_len) == 0)
! *isnull = true;
! else
! *isnull = false;
! return attr_buf.data;
! }
/*
* Send text representation of one attribute, with conversion and escaping
***************
*** 2800,2802 ****
--- 3329,3432 ----
return attnums;
}
+ + /*+ * The following are custom versions of the string function strchr().+ * As opposed to the original strchr which searches through+ * a string until the target character is found, or a NULL is+ * found, this version will not return when a NULL is found.+ * Instead it will search through a pre-defined length of+ * bytes and will return only if the target character(s) is reached.+ *+ * If our client encoding is not a supported server encoding, we+ * know that it is not safe to look at each character as trailing+ * byte in a multibyte character may be a 7-bit ASCII equivalent.+ * Therefore we use pg_encoding_mblen to skip to the end of the+ * character.+ *+ * returns:+ * pointer to c - if c is located within the string.+ * NULL - if c was not found in specified length of search. Note:+ * this DOESN'T mean that a '\0' was reached.+ */+ char *+ scanTextLine(const char *s, char eol, size_t len)+ {+ const char *start;+ + if (client_encoding_only && !line_buf_converted)+ {+ int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);+ + for (start = s; *s != eol && s < start + len; s += mblen)+ mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);+ + return ((*s == eol) ? (char *) s : NULL);+ }+ else+ return memchr(s,eol,len);+ }+ + + char *+ scanCSVLine(const char *s, char eol, char escapec, char quotec, size_t len)+ {+ const char *start;+ + if (client_encoding_only && !line_buf_converted)+ {+ int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);+ + for (start = s; *s != eol && s < start + len; s += mblen)+ {+ if (in_quote && *s == escapec)+ last_was_esc = !last_was_esc;+ if (*s == quotec && !last_was_esc)+ in_quote = !in_quote;+ if (*s != escapec)+ last_was_esc = false;+ + mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);+ }+ }+ else+ /* safe to scroll byte by byte */+ {+ for (start = s; *s != eol && s < start + len; s++)+ {+ if (in_quote && *s == escapec)+ last_was_esc = !last_was_esc;+ if (*s == quotec && !last_was_esc)+ in_quote = !in_quote;+ if (*s != escapec)+ last_was_esc = false;+ }+ + }+ + if(*s != escapec) /* found eol_ch */+ last_was_esc = false;+ + return ((*s == eol) ? (char *) s : NULL);+ }+ + /*+ * Scan for 1 of 2 characters necessary for attribute parsing. No need+ * for checking for multi-byte characters since conversion already + * happened.+ */+ char *+ scanTextAttr(const char *s, char c1, char c2, size_t len)+ {+ const char *start;+ + for (start = s; *s != c1 && *s != c2 && s < start + len; s++)+ ;+ + return (*s != '\0' ? (char *) s : NULL);+ }+ + + + +
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-performance@postgresql.org
Cc: agoldshuv@greenplum.com, pgsql-patches@postgresql.org
Subject: Re: COPY FROM performance improvements
In-Reply-To: <BEFC4FCB.7017%agoldshuv@greenplum.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox