agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v7 1/4] Introduce a new syntax to add/drop publications
35+ messages / 2 participants
[nested] [flat]

* [PATCH v7 1/4] Introduce a new syntax to add/drop publications
@ 2021-03-07 12:56 Japin Li <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Japin Li @ 2021-03-07 12:56 UTC (permalink / raw)

At present, if we want to update publications in subscription, we can
use SET PUBLICATION, however, it requires supply all publications that
exists and the new publications if we want to add new publications, it's
inconvenient.  The new syntax only supply the new publications.  When
the refresh is true, it only refresh the new publications.
---
 src/backend/commands/subscriptioncmds.c | 132 ++++++++++++++++++++++++
 src/backend/parser/gram.y               |  20 ++++
 src/include/nodes/parsenodes.h          |   2 +
 3 files changed, 154 insertions(+)

diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bfd3514546..bf1e579e6f 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -47,6 +47,7 @@
 #include "utils/syscache.h"
 
 static List *fetch_table_list(WalReceiverConn *wrconn, List *publications);
+static List *merge_publications(List *oldpublist, List *newpublist, bool addpub);
 static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
 
 
@@ -964,6 +965,53 @@ AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel)
 				break;
 			}
 
+		case ALTER_SUBSCRIPTION_ADD_PUBLICATION:
+		case ALTER_SUBSCRIPTION_DROP_PUBLICATION:
+			{
+				bool	copy_data = false;
+				bool	isadd = stmt->kind == ALTER_SUBSCRIPTION_ADD_PUBLICATION;
+				bool	refresh;
+				List   *publist = NIL;
+
+				publist = merge_publications(sub->publications, stmt->publication, isadd);
+
+				parse_subscription_options(stmt->options,
+										   NULL,	/* no "connect" */
+										   NULL, NULL,	/* no "enabled" */
+										   NULL,	/* no "create_slot" */
+										   NULL, NULL,	/* no "slot_name" */
+										   isadd ? &copy_data : NULL, /* for drop, no "copy_data" */
+										   NULL,	/* no "synchronous_commit" */
+										   &refresh,
+										   NULL, NULL,	/* no "binary" */
+										   NULL, NULL); /* no "streaming" */
+
+				values[Anum_pg_subscription_subpublications - 1] =
+					publicationListToArray(publist);
+				replaces[Anum_pg_subscription_subpublications - 1] = true;
+
+				update_tuple = true;
+
+				/* Refresh if user asked us to. */
+				if (refresh)
+				{
+					if (!sub->enabled)
+						ereport(ERROR,
+								(errcode(ERRCODE_SYNTAX_ERROR),
+								 errmsg("ALTER SUBSCRIPTION with refresh is not allowed for disabled subscriptions"),
+								 errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION ... WITH (refresh = false).")));
+
+					PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh");
+
+					/* Only refresh the added/dropped list of publications. */
+					sub->publications = stmt->publication;
+
+					AlterSubscription_refresh(sub, copy_data);
+				}
+
+				break;
+			}
+
 		case ALTER_SUBSCRIPTION_REFRESH:
 			{
 				bool		copy_data;
@@ -1551,3 +1599,87 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 			 errhint("Use %s to disassociate the subscription from the slot.",
 					 "ALTER SUBSCRIPTION ... SET (slot_name = NONE)")));
 }
+
+/*
+ * Merge current subscription's publications and user specified publications
+ * by ADD/DROP PUBLICATIONS.
+ *
+ * If addpub is true, we will add the list of publications into oldpublist.
+ * Otherwise, we will delete the list of publications from oldpublist.
+ */
+static List *
+merge_publications(List *oldpublist, List *newpublist, bool addpub)
+{
+	StringInfoData	errstr;
+	int				errstrcnt = 0;
+	ListCell	   *lc;
+
+	initStringInfo(&errstr);
+
+	foreach(lc, newpublist)
+	{
+		char		*name = strVal(lfirst(lc));
+		ListCell	*cell = NULL;
+
+		foreach(cell, oldpublist)
+		{
+			char	*pubname = strVal(lfirst(cell));
+
+			if (strcmp(name, pubname) == 0)
+			{
+				if (addpub)
+				{
+					errstrcnt++;
+
+					if (errstrcnt == 1)
+						appendStringInfo(&errstr, _("\"%s\""), name);
+					else
+						appendStringInfo(&errstr, _(", \"%s\""), name);
+				}
+				else
+					oldpublist = list_delete_cell(oldpublist, cell);
+
+				break;
+			}
+		}
+
+		if (addpub && cell == NULL)
+			oldpublist = lappend(oldpublist, makeString(name));
+		else if (!addpub && cell == NULL)
+		{
+			errstrcnt++;
+
+			if (errstrcnt == 1)
+				appendStringInfo(&errstr, _("\"%s\""), name);
+			else
+				appendStringInfo(&errstr, _(", \"%s\""), name);
+		}
+	}
+
+	if (errstrcnt >= 1)
+	{
+		if (addpub)
+		{
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					 errmsg_plural("publication %s is already present in the subscription",
+								   "publications %s are already present in the subscription",
+								   errstrcnt, errstr.data)));
+		}
+		else
+		{
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					 errmsg_plural("publication %s doesn't exist in the subscription",
+								   "publications %s do not exist in the subscription",
+								   errstrcnt, errstr.data)));
+		}
+	}
+
+	if (oldpublist == NIL)
+		ereport(ERROR,
+				(errcode(ERRCODE_SYNTAX_ERROR),
+				 errmsg("subscription must contain at least one publication")));
+
+	return oldpublist;
+}
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 652be0b96d..6d75e82096 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -9609,6 +9609,26 @@ AlterSubscriptionStmt:
 					n->options = $6;
 					$$ = (Node *)n;
 				}
+			| ALTER SUBSCRIPTION name ADD_P PUBLICATION name_list opt_definition
+				{
+					AlterSubscriptionStmt *n =
+						makeNode(AlterSubscriptionStmt);
+					n->kind = ALTER_SUBSCRIPTION_ADD_PUBLICATION;
+					n->subname = $3;
+					n->publication = $6;
+					n->options = $7;
+					$$ = (Node *)n;
+				}
+			| ALTER SUBSCRIPTION name DROP PUBLICATION name_list opt_definition
+				{
+					AlterSubscriptionStmt *n =
+						makeNode(AlterSubscriptionStmt);
+					n->kind = ALTER_SUBSCRIPTION_DROP_PUBLICATION;
+					n->subname = $3;
+					n->publication = $6;
+					n->options = $7;
+					$$ = (Node *)n;
+				}
 			| ALTER SUBSCRIPTION name SET PUBLICATION name_list opt_definition
 				{
 					AlterSubscriptionStmt *n =
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 236832a2ca..e109607936 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -3580,6 +3580,8 @@ typedef enum AlterSubscriptionType
 	ALTER_SUBSCRIPTION_OPTIONS,
 	ALTER_SUBSCRIPTION_CONNECTION,
 	ALTER_SUBSCRIPTION_PUBLICATION,
+	ALTER_SUBSCRIPTION_ADD_PUBLICATION,
+	ALTER_SUBSCRIPTION_DROP_PUBLICATION,
 	ALTER_SUBSCRIPTION_REFRESH,
 	ALTER_SUBSCRIPTION_ENABLED
 } AlterSubscriptionType;
-- 
2.25.1


--=-=-=
Content-Type: text/x-patch
Content-Disposition: attachment;
 filename=v7-0002-Test-ALTER-SUBSCRIPTION-.-ADD-DROP-PUBLICATION.patch



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension
@ 2024-01-23 06:12 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
  method for CopyToState and it's used for flushing. End-of-row related
  codes were moved to CopyToTextSendEndOfRow().
---
 src/backend/commands/copyto.c  | 15 +++++++--------
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cfc74ee7b1..b5d8678394 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate);
 static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
 static void CopySendString(CopyToState cstate, const char *str);
 static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
 static void CopySendInt32(CopyToState cstate, int32 val);
 static void CopySendInt16(CopyToState cstate, int16 val);
 
@@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
 		default:
 			break;
 	}
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 static void
@@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
 		}
 	}
 
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 static void
@@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate)
 	/* Generate trailer for a binary copy */
 	CopySendInt16(cstate, -1);
 	/* Need to flush out the trailer */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 CopyToRoutine CopyToRoutineText = {
@@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate)
  * CopySendData sends output data to the destination (file or frontend)
  * CopySendString does the same for null-terminated strings
  * CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- *	(data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ *	(data is not actually flushed except by CopyToStateFlush)
  *
  * NB: no data conversion is applied by these functions
  *----------
@@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c)
 	appendStringInfoCharMacro(cstate->fe_msgbuf, c);
 }
 
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
 {
 	StringInfo	fe_msgbuf = cstate->fe_msgbuf;
 
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index a869d78d72..ffad433a21 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -174,6 +174,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.43.0


----Next_Part(Wed_Jan_24_14_49_36_2024_411)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v6-0005-Extract-COPY-FROM-format-implementations.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ccfbacb4a37..4fa23d992f5 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index fb68f42ce1e..93b041352c5 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 206d4c9fac9..2de610ef729 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -302,8 +302,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -405,6 +410,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v23-0001-Add-CopyToRountine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ec86a17b3b3..64772877b0f 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 37b150b44ba..c99edae575b 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 8a560903ede..c1e9fe366f3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -402,6 +407,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v21-0001-Add-CopyToRountine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 74844103228..a115d7f9e26 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 54aa6cdecaf..b8d0e996117 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -500,6 +500,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index e298b19860c..5665408eaa0 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -402,6 +407,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Jul_30_16_13_06_2024_473)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
  method for CopyToState and it's used for flushing. End-of-row related
  codes were moved to CopyToTextSendEndOfRow().

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
  method for CopyFromState and "BinaryData" is redundant.
---
 src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
 src/backend/commands/copyto.c        | 15 +++++++--------
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 74844103228..cd80d34f3da 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -164,7 +164,6 @@ static int	CopyGetData(CopyFromState cstate, void *databuf,
 static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
 static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
 static void CopyLoadInputBuf(CopyFromState cstate);
-static int	CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
 
 void
 ReceiveCopyBegin(CopyFromState cstate)
@@ -193,7 +192,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	int32		tmp;
 
 	/* Signature */
-	if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+	if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
 		memcmp(readSig, BinarySignature, 11) != 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -221,7 +220,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	/* Skip extension header, if present */
 	while (tmp-- > 0)
 	{
-		if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+		if (CopyFromStateRead(cstate, readSig, 1) != 1)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("invalid COPY file header (wrong length)")));
@@ -363,7 +362,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
 {
 	uint32		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -380,7 +379,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
 {
 	uint16		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -691,14 +690,14 @@ CopyLoadInputBuf(CopyFromState cstate)
 }
 
 /*
- * CopyReadBinaryData
+ * CopyFromStateRead
  *
  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
  * and writes them to 'dest'.  Returns the number of bytes read (which
  * would be less than 'nbytes' only if we reach EOF).
  */
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
 {
 	int			copied_bytes = 0;
 
@@ -1078,7 +1077,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext,
 		 */
 		char		dummy;
 
-		if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+		if (CopyFromStateRead(cstate, &dummy, 1) > 0)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("received copy data after EOF marker")));
@@ -2103,8 +2102,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
 	resetStringInfo(&cstate->attribute_buf);
 
 	enlargeStringInfo(&cstate->attribute_buf, fld_size);
-	if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
-						   fld_size) != fld_size)
+	if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+						  fld_size) != fld_size)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 				 errmsg("unexpected EOF in COPY data")));
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 54aa6cdecaf..cd9e352533a 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -63,7 +63,6 @@ static void SendCopyEnd(CopyToState cstate);
 static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
 static void CopySendString(CopyToState cstate, const char *str);
 static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
 static void CopySendInt32(CopyToState cstate, int32 val);
 static void CopySendInt16(CopyToState cstate, int16 val);
 
@@ -99,7 +98,7 @@ CopyToTextLikeSendEndOfRow(CopyToState cstate)
 	}
 
 	/* Now take the actions related to the end of a row */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -325,7 +324,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
 		}
 	}
 
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -339,7 +338,7 @@ CopyToBinaryEnd(CopyToState cstate)
 	/* Generate trailer for a binary copy */
 	CopySendInt16(cstate, -1);
 	/* Need to flush out the trailer */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -419,8 +418,8 @@ SendCopyEnd(CopyToState cstate)
  * CopySendData sends output data to the destination (file or frontend)
  * CopySendString does the same for null-terminated strings
  * CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- *	(data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ *	(data is not actually flushed except by CopyToStateFlush)
  *
  * NB: no data conversion is applied by these functions
  *----------
@@ -443,8 +442,8 @@ CopySendChar(CopyToState cstate, char c)
 	appendStringInfoCharMacro(cstate->fe_msgbuf, c);
 }
 
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
 {
 	StringInfo	fe_msgbuf = cstate->fe_msgbuf;
 
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 3104d99ea9f..0820b47a2d2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -401,6 +406,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Wed_Jul_24_17_30_59_2024_070)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension
@ 2024-01-24 05:19 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-24 05:19 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
  method for CopyFromState and "BinaryData" is redundant.
---
 src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index a78a790060..f8a194635d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -165,7 +165,6 @@ static int	CopyGetData(CopyFromState cstate, void *databuf,
 static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
 static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
 static void CopyLoadInputBuf(CopyFromState cstate);
-static int	CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
 
 void
 ReceiveCopyBegin(CopyFromState cstate)
@@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	int32		tmp;
 
 	/* Signature */
-	if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+	if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
 		memcmp(readSig, BinarySignature, 11) != 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	/* Skip extension header, if present */
 	while (tmp-- > 0)
 	{
-		if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+		if (CopyFromStateRead(cstate, readSig, 1) != 1)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("invalid COPY file header (wrong length)")));
@@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
 {
 	uint32		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
 {
 	uint16		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate)
 }
 
 /*
- * CopyReadBinaryData
+ * CopyFromStateRead
  *
  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
  * and writes them to 'dest'.  Returns the number of bytes read (which
  * would be less than 'nbytes' only if we reach EOF).
  */
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
 {
 	int			copied_bytes = 0;
 
@@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 		 */
 		char		dummy;
 
-		if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+		if (CopyFromStateRead(cstate, &dummy, 1) > 0)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("received copy data after EOF marker")));
@@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
 	resetStringInfo(&cstate->attribute_buf);
 
 	enlargeStringInfo(&cstate->attribute_buf, fld_size);
-	if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
-						   fld_size) != fld_size)
+	if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+						  fld_size) != fld_size)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 				 errmsg("unexpected EOF in COPY data")));
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b7e8f627bf..22accc83ab 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -314,8 +314,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 /*
  * Represents the different dest cases we need to worry about at
  * the bottom level
-- 
2.43.0


----Next_Part(Wed_Jan_24_14_49_36_2024_411)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension
@ 2024-09-28 14:59 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 14 ++++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index fb68f42ce1e..93b041352c5 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b6ddb5f6216..310a37ba728 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -190,6 +190,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v23-0006-Add-CopyFromRoutine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension
@ 2024-09-28 14:59 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 14 ++++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 37b150b44ba..c99edae575b 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 03779c15f43..30765951e2e 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -187,6 +187,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v21-0006-Add-CopyFromRoutine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension
@ 2024-09-28 15:32 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/include/commands/copyapi.h       |  6 ++++++
 2 files changed, 20 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ec86a17b3b3..64772877b0f 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c118558ee71..c1e9fe366f3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,14 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
+
 typedef struct CopyToStateData *CopyToState;
 
 /*
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension
@ 2024-09-28 15:32 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/include/commands/copyapi.h       |  6 ++++++
 2 files changed, 20 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ccfbacb4a37..4fa23d992f5 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 0274e3487c3..2de610ef729 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -302,8 +302,14 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
+
 typedef struct CopyToStateData *CopyToState;
 
 /*
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 1e75e07dc0b..7487190bdd6 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.47.1


----Next_Part(Thu_Jan_23_18_12_10_2025_763)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v28-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 16d3b389e97..20d49d73e38 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c125dc3e209..d0da9e07a0d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.1


----Next_Part(Thu_Feb__6_21_06_31_2025_538)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v32-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Wed_Mar__5_09_06_08_2025_350)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v36-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 96b5e144a1d..cb9bfa0053f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.45.2


----Next_Part(Mon_Nov_25_15_01_50_2024_156)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v26-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 91fa46ddf6f..da281f32950 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 5d071b378d6..f8167af4c79 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 2df53dda8a0..4b82372691e 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts);
-- 
2.47.1


----Next_Part(Sat_Feb__1_08_10_23_2025_403)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v30-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Thu_Mar_20_10_24_55_2025_309)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v38-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 16d3b389e97..20d49d73e38 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c125dc3e209..d0da9e07a0d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.1


----Next_Part(Sat_Feb__1_19_12_01_2025_760)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v31-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 91fa46ddf6f..da281f32950 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 5d071b378d6..f8167af4c79 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 2df53dda8a0..4b82372691e 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts);
-- 
2.47.1


----Next_Part(Fri_Jan_31_00_42_13_2025_303)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v29-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Wed_Mar_19_11_56_17_2025_532)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v37-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 96b5e144a1d..cb9bfa0053f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.45.2


----Next_Part(Wed_Nov_27_16_53_44_2024_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v27-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 17d89c23af0..35f9035141a 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Sat_Mar__1_11_50_09_2025_878)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v35-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Thu_Mar_20_10_24_55_2025_309)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v38-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 75b49629f08..01f2e7a8824 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 103eb21767d..ac58adbd23d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -104,4 +104,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Sat_Feb__1_19_12_01_2025_760)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v31-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Sat_Mar__1_11_50_09_2025_878)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v35-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index f7982bf692f..650b6b2382b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index bf933069fea..d1a1dbeb178 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -105,4 +105,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3743b11faa4..a65bbbc962e 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Sat_Feb__1_08_10_23_2025_403)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v30-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 5fcdbea2c2a..d79b6ebe8a3 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 9358515c6f6..6f158272829 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.47.1


----Next_Part(Thu_Jan_23_18_12_10_2025_763)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v28-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index f7982bf692f..650b6b2382b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index bf933069fea..d1a1dbeb178 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -105,4 +105,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3743b11faa4..a65bbbc962e 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Fri_Jan_31_00_42_13_2025_303)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v29-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 1c68b0d2952..0a7e7255b7d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 335584f8877..caba308533d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Mon_Nov_25_15_01_50_2024_156)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 1c68b0d2952..0a7e7255b7d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 335584f8877..caba308533d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Wed_Nov_27_16_53_44_2024_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v27-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Wed_Mar__5_09_06_08_2025_350)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v36-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Wed_Mar_19_11_56_17_2025_532)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v37-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 75b49629f08..01f2e7a8824 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 103eb21767d..ac58adbd23d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -104,4 +104,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Thu_Feb__6_21_06_31_2025_538)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v32-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v39 3/5] Add support for implementing custom COPY handler as extension
@ 2025-03-27 02:24 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw)

* TO: Add CopyToStateData::opaque that can be used to keep
  data for custom COPY TO handler implementation
* TO: Export CopySendEndOfRow() to send end of row data as
  CopyToStateFlush()
* FROM: Add CopyFromStateData::opaque that can be used to
  keep data for custom COPY FROM handler implementation
* FROM: Export CopyGetData() to get the next data as
  CopyFromStateGetData()
* FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and
  "LOG_VERBOSITY verbose"

COPY FROM extensions must call CopyFromSkipErrorRow() when
CopyFromOneRow callback reports an error by
errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and
"LOG_VERBOSITY verbose" cases.
---
 src/backend/commands/copyfromparse.c          | 93 ++++++++++++-------
 src/backend/commands/copyto.c                 | 12 +++
 src/include/commands/copyapi.h                |  6 ++
 src/include/commands/copyfrom_internal.h      |  3 +
 src/include/commands/copyto_internal.h        |  3 +
 .../test_copy_format/expected/no_schema.out   | 47 ++++++++++
 .../test_copy_format/sql/no_schema.sql        | 24 +++++
 .../test_copy_format/test_copy_format.c       | 80 +++++++++++++++-
 8 files changed, 231 insertions(+), 37 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..2070f51a963 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
@@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 	return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
 }
 
+/*
+ * Call this when you report an error by errsave() in your CopyFromOneRow
+ * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases
+ * for you.
+ */
+void
+CopyFromSkipErrorRow(CopyFromState cstate)
+{
+	Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
+
+	cstate->num_errors++;
+
+	if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
+	{
+		/*
+		 * Since we emit line number and column info in the below notice
+		 * message, we suppress error context information other than the
+		 * relation name.
+		 */
+		Assert(!cstate->relname_only);
+		cstate->relname_only = true;
+
+		if (cstate->cur_attval)
+		{
+			char	   *attval;
+
+			attval = CopyLimitPrintoutLength(cstate->cur_attval);
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"",
+						   (unsigned long long) cstate->cur_lineno,
+						   cstate->cur_attname,
+						   attval));
+			pfree(attval);
+		}
+		else
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input",
+						   (unsigned long long) cstate->cur_lineno,
+						   cstate->cur_attname));
+
+		/* reset relname_only */
+		cstate->relname_only = false;
+	}
+}
+
 /*
  * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
  *
@@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
 										(Node *) cstate->escontext,
 										&values[m]))
 		{
-			Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
-
-			cstate->num_errors++;
-
-			if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
-			{
-				/*
-				 * Since we emit line number and column info in the below
-				 * notice message, we suppress error context information other
-				 * than the relation name.
-				 */
-				Assert(!cstate->relname_only);
-				cstate->relname_only = true;
-
-				if (cstate->cur_attval)
-				{
-					char	   *attval;
-
-					attval = CopyLimitPrintoutLength(cstate->cur_attval);
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"",
-								   (unsigned long long) cstate->cur_lineno,
-								   cstate->cur_attname,
-								   attval));
-					pfree(attval);
-				}
-				else
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input",
-								   (unsigned long long) cstate->cur_lineno,
-								   cstate->cur_attname));
-
-				/* reset relname_only */
-				cstate->relname_only = false;
-			}
-
+			CopyFromSkipErrorRow(cstate);
 			return true;
 		}
 
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b7ff6466ce3..23cbdad184c 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -456,6 +456,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 53ad3337f86..500ece7d5bb 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
@@ -106,4 +108,8 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
+extern void CopyFromSkipErrorRow(CopyFromState cstate);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 12c4a0f5979..14ee0f50588 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
diff --git a/src/test/modules/test_copy_format/expected/no_schema.out b/src/test/modules/test_copy_format/expected/no_schema.out
index d5903632b2e..05d160c1eae 100644
--- a/src/test/modules/test_copy_format/expected/no_schema.out
+++ b/src/test/modules/test_copy_format/expected/no_schema.out
@@ -1,6 +1,8 @@
 CREATE EXTENSION test_copy_format;
 CREATE TABLE public.test (a smallint, b integer, c bigint);
 INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=true
 NOTICE:  CopyFromInFunc: attribute: smallint
@@ -8,7 +10,50 @@ NOTICE:  CopyFromInFunc: attribute: integer
 NOTICE:  CopyFromInFunc: attribute: bigint
 NOTICE:  CopyFromStart: the number of attributes: 3
 NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  invalid value: "6"
+CONTEXT:  COPY test, line 2, column a: "6"
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
 NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  skipping row due to data type incompatibility at line 2 for column "a": "6"
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
+NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  too much lines: 3
+CONTEXT:  COPY test, line 3
 COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=false
 NOTICE:  CopyToOutFunc: attribute: smallint
@@ -18,6 +63,8 @@ NOTICE:  CopyToStart: the number of attributes: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToEnd
 DROP TABLE public.test;
 DROP EXTENSION test_copy_format;
diff --git a/src/test/modules/test_copy_format/sql/no_schema.sql b/src/test/modules/test_copy_format/sql/no_schema.sql
index 1e049f799f0..1901c4a9f43 100644
--- a/src/test/modules/test_copy_format/sql/no_schema.sql
+++ b/src/test/modules/test_copy_format/sql/no_schema.sql
@@ -1,7 +1,31 @@
 CREATE EXTENSION test_copy_format;
 CREATE TABLE public.test (a smallint, b integer, c bigint);
 INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+321
 \.
 COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
 DROP TABLE public.test;
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 1d754201336..34ec693a7ec 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -14,6 +14,7 @@
 #include "postgres.h"
 
 #include "commands/copyapi.h"
+#include "commands/copyfrom_internal.h"
 #include "commands/defrem.h"
 #include "utils/builtins.h"
 
@@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
 static bool
 TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 {
+	int			n_attributes = list_length(cstate->attnumlist);
+	char	   *line;
+	int			line_size = n_attributes + 1;	/* +1 is for new line */
+	int			read_bytes;
+
 	ereport(NOTICE, (errmsg("CopyFromOneRow")));
-	return false;
+
+	cstate->cur_lineno++;
+	line = palloc(line_size);
+	read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size);
+	if (read_bytes == 0)
+		return false;
+	if (read_bytes != line_size)
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("one line must be %d bytes: %d",
+						line_size, read_bytes)));
+
+	if (cstate->cur_lineno == 1)
+	{
+		/* Success */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		ListCell   *cur;
+		int			i = 0;
+
+		foreach(cur, cstate->attnumlist)
+		{
+			int			attnum = lfirst_int(cur);
+			int			m = attnum - 1;
+			Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+			if (att->atttypid == INT2OID)
+			{
+				values[i] = Int16GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT4OID)
+			{
+				values[i] = Int32GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT8OID)
+			{
+				values[i] = Int64GetDatum(line[i] - '0');
+			}
+			nulls[i] = false;
+			i++;
+		}
+	}
+	else if (cstate->cur_lineno == 2)
+	{
+		/* Soft error */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		int			attnum = lfirst_int(list_head(cstate->attnumlist));
+		int			m = attnum - 1;
+		Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+		char		value[2];
+
+		cstate->cur_attname = NameStr(att->attname);
+		value[0] = line[0];
+		value[1] = '\0';
+		cstate->cur_attval = value;
+		errsave((Node *) cstate->escontext,
+				(
+				 errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("invalid value: \"%c\"", line[0])));
+		CopyFromSkipErrorRow(cstate);
+		cstate->cur_attname = NULL;
+		cstate->cur_attval = NULL;
+		return true;
+	}
+	else
+	{
+		/* Hard error */
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("too much lines: %llu",
+						(unsigned long long) cstate->cur_lineno)));
+	}
+
+	return true;
 }
 
 static void
-- 
2.47.2


----Next_Part(Thu_Mar_27_12_28_40_2025_510)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v39-0004-Use-copy-handlers-for-built-in-formats.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v40 3/6] Add support for implementing custom COPY handler as extension
@ 2025-03-27 02:24 Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw)

* TO: Add CopyToStateData::opaque that can be used to keep
  data for custom COPY TO handler implementation
* TO: Export CopySendEndOfRow() to send end of row data as
  CopyToStateFlush()
* FROM: Add CopyFromStateData::opaque that can be used to
  keep data for custom COPY FROM handler implementation
* FROM: Export CopyGetData() to get the next data as
  CopyFromStateGetData()
* FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and
  "LOG_VERBOSITY verbose"

COPY FROM extensions must call CopyFromSkipErrorRow() when
CopyFromOneRow callback reports an error by
errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and
"LOG_VERBOSITY verbose" cases.
---
 src/backend/commands/copyfromparse.c          | 93 ++++++++++++-------
 src/backend/commands/copyto.c                 | 12 +++
 src/include/commands/copyapi.h                |  6 ++
 src/include/commands/copyfrom_internal.h      |  3 +
 src/include/commands/copyto_internal.h        |  3 +
 .../expected/test_copy_format.out             | 50 ++++++++++
 .../test_copy_format/sql/test_copy_format.sql | 35 +++++++
 .../test_copy_format/test_copy_format.c       | 80 +++++++++++++++-
 8 files changed, 245 insertions(+), 37 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 9f7171d1478..de68b53b000 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
@@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 	return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
 }
 
+/*
+ * Call this when you report an error by errsave() in your CopyFromOneRow
+ * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases
+ * for you.
+ */
+void
+CopyFromSkipErrorRow(CopyFromState cstate)
+{
+	Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
+
+	cstate->num_errors++;
+
+	if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
+	{
+		/*
+		 * Since we emit line number and column info in the below notice
+		 * message, we suppress error context information other than the
+		 * relation name.
+		 */
+		Assert(!cstate->relname_only);
+		cstate->relname_only = true;
+
+		if (cstate->cur_attval)
+		{
+			char	   *attval;
+
+			attval = CopyLimitPrintoutLength(cstate->cur_attval);
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
+						   cstate->cur_lineno,
+						   cstate->cur_attname,
+						   attval));
+			pfree(attval);
+		}
+		else
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
+						   cstate->cur_lineno,
+						   cstate->cur_attname));
+
+		/* reset relname_only */
+		cstate->relname_only = false;
+	}
+}
+
 /*
  * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
  *
@@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
 										(Node *) cstate->escontext,
 										&values[m]))
 		{
-			Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
-
-			cstate->num_errors++;
-
-			if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
-			{
-				/*
-				 * Since we emit line number and column info in the below
-				 * notice message, we suppress error context information other
-				 * than the relation name.
-				 */
-				Assert(!cstate->relname_only);
-				cstate->relname_only = true;
-
-				if (cstate->cur_attval)
-				{
-					char	   *attval;
-
-					attval = CopyLimitPrintoutLength(cstate->cur_attval);
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
-								   cstate->cur_lineno,
-								   cstate->cur_attname,
-								   attval));
-					pfree(attval);
-				}
-				else
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
-								   cstate->cur_lineno,
-								   cstate->cur_attname));
-
-				/* reset relname_only */
-				cstate->relname_only = false;
-			}
-
+			CopyFromSkipErrorRow(cstate);
 			return true;
 		}
 
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 265b847e255..d6fcfdfb9b1 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -454,6 +454,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 53ad3337f86..500ece7d5bb 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
@@ -106,4 +108,8 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
+extern void CopyFromSkipErrorRow(CopyFromState cstate);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 24157e11a73..f9e27152313 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index da796131988..3bd9d702bf0 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
index 3916b766615..47a875f0ab1 100644
--- a/src/test/modules/test_copy_format/expected/test_copy_format.out
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -4,6 +4,8 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 -- schema.
 CREATE EXTENSION test_copy_format;
 -- We can find a custom COPY handler without schema.
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=true
 NOTICE:  CopyFromInFunc: attribute: smallint
@@ -11,7 +13,50 @@ NOTICE:  CopyFromInFunc: attribute: integer
 NOTICE:  CopyFromInFunc: attribute: bigint
 NOTICE:  CopyFromStart: the number of attributes: 3
 NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  invalid value: "6"
+CONTEXT:  COPY copy_data, line 2, column a: "6"
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
 NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  skipping row due to data type incompatibility at line 2 for column "a": "6"
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
+NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  too much lines: 3
+CONTEXT:  COPY copy_data, line 3
 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=false
 NOTICE:  CopyToOutFunc: attribute: smallint
@@ -21,7 +66,12 @@ NOTICE:  CopyToStart: the number of attributes: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToEnd
+-- Reset data.
+TRUNCATE copy_data;
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 DROP EXTENSION test_copy_format;
 -- Install custom COPY handlers to a schema that isn't included in
 -- search_path.
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
index b262794f878..c7beb2fb8ae 100644
--- a/src/test/modules/test_copy_format/sql/test_copy_format.sql
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -4,10 +4,45 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 -- No WITH SCHEMA. It installs custom COPY handlers to the current
 -- schema.
 CREATE EXTENSION test_copy_format;
+
 -- We can find a custom COPY handler without schema.
+
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format');
+987
+654
 \.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+\.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+987
+654
+\.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+321
+\.
+
 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format');
+
+-- Reset data.
+TRUNCATE copy_data;
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+
 DROP EXTENSION test_copy_format;
 
 
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 1d754201336..34ec693a7ec 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -14,6 +14,7 @@
 #include "postgres.h"
 
 #include "commands/copyapi.h"
+#include "commands/copyfrom_internal.h"
 #include "commands/defrem.h"
 #include "utils/builtins.h"
 
@@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
 static bool
 TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 {
+	int			n_attributes = list_length(cstate->attnumlist);
+	char	   *line;
+	int			line_size = n_attributes + 1;	/* +1 is for new line */
+	int			read_bytes;
+
 	ereport(NOTICE, (errmsg("CopyFromOneRow")));
-	return false;
+
+	cstate->cur_lineno++;
+	line = palloc(line_size);
+	read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size);
+	if (read_bytes == 0)
+		return false;
+	if (read_bytes != line_size)
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("one line must be %d bytes: %d",
+						line_size, read_bytes)));
+
+	if (cstate->cur_lineno == 1)
+	{
+		/* Success */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		ListCell   *cur;
+		int			i = 0;
+
+		foreach(cur, cstate->attnumlist)
+		{
+			int			attnum = lfirst_int(cur);
+			int			m = attnum - 1;
+			Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+			if (att->atttypid == INT2OID)
+			{
+				values[i] = Int16GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT4OID)
+			{
+				values[i] = Int32GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT8OID)
+			{
+				values[i] = Int64GetDatum(line[i] - '0');
+			}
+			nulls[i] = false;
+			i++;
+		}
+	}
+	else if (cstate->cur_lineno == 2)
+	{
+		/* Soft error */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		int			attnum = lfirst_int(list_head(cstate->attnumlist));
+		int			m = attnum - 1;
+		Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+		char		value[2];
+
+		cstate->cur_attname = NameStr(att->attname);
+		value[0] = line[0];
+		value[1] = '\0';
+		cstate->cur_attval = value;
+		errsave((Node *) cstate->escontext,
+				(
+				 errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("invalid value: \"%c\"", line[0])));
+		CopyFromSkipErrorRow(cstate);
+		cstate->cur_attname = NULL;
+		cstate->cur_attval = NULL;
+		return true;
+	}
+	else
+	{
+		/* Hard error */
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("too much lines: %llu",
+						(unsigned long long) cstate->cur_lineno)));
+	}
+
+	return true;
 }
 
 static void
-- 
2.47.2


----Next_Part(Fri_Apr_25_21_45_34_2025_836)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v40-0004-Use-copy-handlers-for-built-in-formats.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

end of thread, other threads:[~2025-03-27 02:24 UTC | newest]

Thread overview: 35+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-07 12:56 [PATCH v7 1/4] Introduce a new syntax to add/drop publications Japin Li <[email protected]>
2024-01-23 06:12 [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-24 05:19 [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-09-28 14:59 [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-09-28 14:59 [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-09-28 15:32 [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-09-28 15:32 [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2025-03-27 02:24 [PATCH v39 3/5] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>
2025-03-27 02:24 [PATCH v40 3/6] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox