agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v3] Allow a partdesc-omitting-partitions to be cached
35+ messages / 2 participants
[nested] [flat]

* [PATCH v3] Allow a partdesc-omitting-partitions to be cached
@ 2021-04-26 18:53  Alvaro Herrera <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Alvaro Herrera @ 2021-04-26 18:53 UTC (permalink / raw)

Makes partition descriptor acquisition faster during the transient
period in which a partition is in the process of being detached.

Per gripe from Amit Langote
Discussion: https://postgr.es/m/CA+HiwqFgpP1LxJZOBYGt9rpvTjXXkg5qG2+Xch2Z1Q7KrqZR1A@mail.gmail.com
---
 src/backend/catalog/pg_inherits.c             |  52 ++++++++-
 src/backend/commands/tablecmds.c              |  66 +++++++-----
 src/backend/commands/trigger.c                |   3 +-
 src/backend/partitioning/partdesc.c           | 101 ++++++++++++------
 src/backend/utils/cache/relcache.c            |  33 +++++-
 src/include/catalog/pg_inherits.h             |   6 +-
 src/include/utils/rel.h                       |   5 +
 .../detach-partition-concurrently-3.out       |  57 +++++++++-
 .../detach-partition-concurrently-3.spec      |   9 +-
 9 files changed, 254 insertions(+), 78 deletions(-)

diff --git a/src/backend/catalog/pg_inherits.c b/src/backend/catalog/pg_inherits.c
index 6447b52854..c373faf2d6 100644
--- a/src/backend/catalog/pg_inherits.c
+++ b/src/backend/catalog/pg_inherits.c
@@ -52,6 +52,22 @@ typedef struct SeenRelsEntry
  * then no locks are acquired, but caller must beware of race conditions
  * against possible DROPs of child relations.
  *
+ * Partitions marked as being detached are omitted; see
+ * find_inheritance_children_extended for details.
+ */
+List *
+find_inheritance_children(Oid parentrelId, LOCKMODE lockmode)
+{
+	return find_inheritance_children_extended(parentrelId, true, lockmode,
+											  NULL, NULL);
+}
+
+/*
+ * find_inheritance_children_extended
+ *
+ * As find_inheritance_children, with more options regarding detached
+ * partitions.
+ *
  * If a partition's pg_inherits row is marked "detach pending",
  * *detached_exist (if not null) is set true.
  *
@@ -60,11 +76,13 @@ typedef struct SeenRelsEntry
  * marked "detach pending" is visible to that snapshot, then that partition is
  * omitted from the output list.  This makes partitions invisible depending on
  * whether the transaction that marked those partitions as detached appears
- * committed to the active snapshot.
+ * committed to the active snapshot.  In addition, *detached_xmin (if not null)
+ * is set to the xmin of the row of the detached partition.
  */
 List *
-find_inheritance_children(Oid parentrelId, bool omit_detached,
-						  LOCKMODE lockmode, bool *detached_exist)
+find_inheritance_children_extended(Oid parentrelId, bool omit_detached,
+								   LOCKMODE lockmode, bool *detached_exist,
+								   TransactionId *detached_xmin)
 {
 	List	   *list = NIL;
 	Relation	relation;
@@ -132,7 +150,32 @@ find_inheritance_children(Oid parentrelId, bool omit_detached,
 				snap = GetActiveSnapshot();
 
 				if (!XidInMVCCSnapshot(xmin, snap))
+				{
+					if (detached_xmin)
+					{
+						/*
+						 * Two detached partitions should not occur (see
+						 * checks in MarkInheritDetached), but if they do,
+						 * track the newer of the two.  Make sure to warn the
+						 * user, so that they can clean up.  Since this is
+						 * just a cross-check against potentially corrupt
+						 * catalogs, we don't make it a full-fledged error
+						 * message.
+						 */
+						if (*detached_xmin != InvalidTransactionId)
+						{
+							elog(WARNING, "more than one partition pending detach found for table with OID %u",
+								 parentrelId);
+							if (TransactionIdFollows(xmin, *detached_xmin))
+								*detached_xmin = xmin;
+						}
+						else
+							*detached_xmin = xmin;
+					}
+
+					/* Don't add the partition to the output list */
 					continue;
+				}
 			}
 		}
 
@@ -247,8 +290,7 @@ find_all_inheritors(Oid parentrelId, LOCKMODE lockmode, List **numparents)
 		ListCell   *lc;
 
 		/* Get the direct children of this rel */
-		currentchildren = find_inheritance_children(currentrel, true,
-													lockmode, NULL);
+		currentchildren = find_inheritance_children(currentrel, lockmode);
 
 		/*
 		 * Add to the queue only those children not already seen. This avoids
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 8e717ada28..d9ba87a2a3 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -3492,7 +3492,7 @@ renameatt_internal(Oid myrelid,
 		 * expected_parents will only be 0 if we are not already recursing.
 		 */
 		if (expected_parents == 0 &&
-			find_inheritance_children(myrelid, true, NoLock, NULL) != NIL)
+			find_inheritance_children(myrelid, NoLock) != NIL)
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_TABLE_DEFINITION),
 					 errmsg("inherited column \"%s\" must be renamed in child tables too",
@@ -3691,7 +3691,7 @@ rename_constraint_internal(Oid myrelid,
 		else
 		{
 			if (expected_parents == 0 &&
-				find_inheritance_children(myrelid, true, NoLock, NULL) != NIL)
+				find_inheritance_children(myrelid, NoLock) != NIL)
 				ereport(ERROR,
 						(errcode(ERRCODE_INVALID_TABLE_DEFINITION),
 						 errmsg("inherited constraint \"%s\" must be renamed in child tables too",
@@ -6565,7 +6565,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
 	 */
 	if (colDef->identity &&
 		recurse &&
-		find_inheritance_children(myrelid, true, NoLock, NULL) != NIL)
+		find_inheritance_children(myrelid, NoLock) != NIL)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TABLE_DEFINITION),
 				 errmsg("cannot recursively add identity column to table that has child tables")));
@@ -6811,7 +6811,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
 	 * use find_all_inheritors to do it in one pass.
 	 */
 	children =
-		find_inheritance_children(RelationGetRelid(rel), true, lockmode, NULL);
+		find_inheritance_children(RelationGetRelid(rel), lockmode);
 
 	/*
 	 * If we are told not to recurse, there had better not be any child
@@ -7674,7 +7674,7 @@ ATPrepDropExpression(Relation rel, AlterTableCmd *cmd, bool recurse, bool recurs
 	 * resulting state can be properly dumped and restored.
 	 */
 	if (!recurse &&
-		find_inheritance_children(RelationGetRelid(rel), true, lockmode, NULL))
+		find_inheritance_children(RelationGetRelid(rel), lockmode))
 		ereport(ERROR,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("ALTER TABLE / DROP EXPRESSION must be applied to child tables too")));
@@ -8282,7 +8282,7 @@ ATExecDropColumn(List **wqueue, Relation rel, const char *colName,
 	 * use find_all_inheritors to do it in one pass.
 	 */
 	children =
-		find_inheritance_children(RelationGetRelid(rel), true, lockmode, NULL);
+		find_inheritance_children(RelationGetRelid(rel), lockmode);
 
 	if (children)
 	{
@@ -8770,7 +8770,7 @@ ATAddCheckConstraint(List **wqueue, AlteredTableInfo *tab, Relation rel,
 	 * use find_all_inheritors to do it in one pass.
 	 */
 	children =
-		find_inheritance_children(RelationGetRelid(rel), true, lockmode, NULL);
+		find_inheritance_children(RelationGetRelid(rel), lockmode);
 
 	/*
 	 * Check if ONLY was specified with ALTER TABLE.  If so, allow the
@@ -11303,8 +11303,7 @@ ATExecDropConstraint(Relation rel, const char *constrName,
 	 * use find_all_inheritors to do it in one pass.
 	 */
 	if (!is_no_inherit_constraint)
-		children = find_inheritance_children(RelationGetRelid(rel), true,
-											 lockmode, NULL);
+		children = find_inheritance_children(RelationGetRelid(rel), lockmode);
 	else
 		children = NIL;
 
@@ -11688,8 +11687,7 @@ ATPrepAlterColumnType(List **wqueue,
 		}
 	}
 	else if (!recursing &&
-			 find_inheritance_children(RelationGetRelid(rel), true,
-									   NoLock, NULL) != NIL)
+			 find_inheritance_children(RelationGetRelid(rel), NoLock) != NIL)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TABLE_DEFINITION),
 				 errmsg("type of inherited column \"%s\" must be changed in child tables too",
@@ -14601,7 +14599,8 @@ ATExecDropInherit(Relation rel, RangeVar *parent, LOCKMODE lockmode)
  * MarkInheritDetached
  *
  * Set inhdetachpending for a partition, for ATExecDetachPartition
- * in concurrent mode.
+ * in concurrent mode.  While at it, verify that no other partition is
+ * already pending detach.
  */
 static void
 MarkInheritDetached(Relation child_rel, Relation parent_rel)
@@ -14617,32 +14616,45 @@ MarkInheritDetached(Relation child_rel, Relation parent_rel)
 	Assert(parent_rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE);
 
 	/*
-	 * Find pg_inherits entries by inhrelid.
+	 * Find pg_inherits entries by inhparent.  (We need to scan them all in
+	 * order to verify that no other partition is pending detach.)
 	 */
 	catalogRelation = table_open(InheritsRelationId, RowExclusiveLock);
 	ScanKeyInit(&key,
-				Anum_pg_inherits_inhrelid,
+				Anum_pg_inherits_inhparent,
 				BTEqualStrategyNumber, F_OIDEQ,
-				ObjectIdGetDatum(RelationGetRelid(child_rel)));
-	scan = systable_beginscan(catalogRelation, InheritsRelidSeqnoIndexId,
+				ObjectIdGetDatum(RelationGetRelid(parent_rel)));
+	scan = systable_beginscan(catalogRelation, InheritsParentIndexId,
 							  true, NULL, 1, &key);
 
 	while (HeapTupleIsValid(inheritsTuple = systable_getnext(scan)))
 	{
-		HeapTuple	newtup;
+		Form_pg_inherits inhForm;
 
-		if (((Form_pg_inherits) GETSTRUCT(inheritsTuple))->inhparent !=
-			RelationGetRelid(parent_rel))
-			elog(ERROR, "bad parent tuple found for partition %u",
-				 RelationGetRelid(child_rel));
+		inhForm = (Form_pg_inherits) GETSTRUCT(inheritsTuple);
+		if (inhForm->inhdetachpending)
+			ereport(ERROR,
+					errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("partition \"%s\" already pending detach in partitioned table \"%s.%s\"",
+						   get_rel_name(inhForm->inhrelid),
+						   get_namespace_name(parent_rel->rd_rel->relnamespace),
+						   RelationGetRelationName(parent_rel)),
+					errhint("Use ALTER TABLE ... DETACH PARTITION ... FINALIZE to complete the detach operation."));
+
+		if (inhForm->inhrelid == RelationGetRelid(child_rel))
+		{
+			HeapTuple	newtup;
 
-		newtup = heap_copytuple(inheritsTuple);
-		((Form_pg_inherits) GETSTRUCT(newtup))->inhdetachpending = true;
+			newtup = heap_copytuple(inheritsTuple);
+			((Form_pg_inherits) GETSTRUCT(newtup))->inhdetachpending = true;
 
-		CatalogTupleUpdate(catalogRelation,
-						   &inheritsTuple->t_self,
-						   newtup);
-		found = true;
+			CatalogTupleUpdate(catalogRelation,
+							   &inheritsTuple->t_self,
+							   newtup);
+			found = true;
+			heap_freetuple(newtup);
+			/* keep looking, to ensure we catch others pending detach */
+		}
 	}
 
 	/* Done */
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index d8393aa4de..f305f8bc0f 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -1141,8 +1141,7 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString,
 			ListCell   *l;
 			List	   *idxs = NIL;
 
-			idxs = find_inheritance_children(indexOid, true,
-											 ShareRowExclusiveLock, NULL);
+			idxs = find_inheritance_children(indexOid, ShareRowExclusiveLock);
 			foreach(l, idxs)
 				childTbls = lappend_oid(childTbls,
 										IndexGetRelation(lfirst_oid(l),
diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c
index 2305dff407..290db0fa35 100644
--- a/src/backend/partitioning/partdesc.c
+++ b/src/backend/partitioning/partdesc.c
@@ -54,6 +54,12 @@ static PartitionDesc RelationBuildPartitionDesc(Relation rel,
 /*
  * RelationGetPartitionDesc -- get partition descriptor, if relation is partitioned
  *
+ * We keep two partdescs in relcache: rd_partdesc includes all partitions
+ * (even those being concurrently marked detached), while rd_partdesc_nodetach
+ * omits (some of) those.  We store the pg_inherits.xmin value for the latter,
+ * to determine whether it can be validly reused in each case, since that
+ * depends on the active snapshot.
+ *
  * Note: we arrange for partition descriptors to not get freed until the
  * relcache entry's refcount goes to zero (see hacks in RelationClose,
  * RelationClearRelation, and RelationBuildPartitionDesc).  Therefore, even
@@ -61,13 +67,6 @@ static PartitionDesc RelationBuildPartitionDesc(Relation rel,
  * for callers to continue to use that pointer as long as (a) they hold the
  * relation open, and (b) they hold a relation lock strong enough to ensure
  * that the data doesn't become stale.
- *
- * The above applies to partition descriptors that are complete regarding
- * partitions concurrently being detached.  When a descriptor that omits
- * partitions being detached is requested (and such partitions are present),
- * said descriptor is not part of relcache and so it isn't freed by
- * invalidations either.  Caller must not use such a descriptor beyond the
- * current Portal.
  */
 PartitionDesc
 RelationGetPartitionDesc(Relation rel, bool omit_detached)
@@ -78,11 +77,36 @@ RelationGetPartitionDesc(Relation rel, bool omit_detached)
 	 * If relcache has a partition descriptor, use that.  However, we can only
 	 * do so when we are asked to include all partitions including detached;
 	 * and also when we know that there are no detached partitions.
+	 *
+	 * If there is no active snapshot, detached partitions aren't omitted
+	 * either, so we can use the cached descriptor too in that case.
 	 */
 	if (likely(rel->rd_partdesc &&
-			   (!rel->rd_partdesc->detached_exist || !omit_detached)))
+			   (!rel->rd_partdesc->detached_exist || !omit_detached ||
+				!ActiveSnapshotSet())))
 		return rel->rd_partdesc;
 
+	/*
+	 * If we're asked to omit detached partitions, we may be able to use a
+	 * cached descriptor too.  We determine that based on the pg_inherits.xmin
+	 * that was saved alongside that descriptor: if the xmin that was not in
+	 * progress for that active snapshot is also not in progress for the
+	 * current active snapshot, then we can use use it.  Otherwise build one
+	 * from scratch.
+	 */
+	if (omit_detached &&
+		rel->rd_partdesc_nodetached &&
+		TransactionIdIsValid(rel->rd_partdesc_nodetached_xmin) &&
+		ActiveSnapshotSet())
+	{
+		Snapshot	activesnap;
+
+		activesnap = GetActiveSnapshot();
+
+		if (!XidInMVCCSnapshot(rel->rd_partdesc_nodetached_xmin, activesnap))
+			return rel->rd_partdesc_nodetached;
+	}
+
 	return RelationBuildPartitionDesc(rel, omit_detached);
 }
 
@@ -117,10 +141,13 @@ RelationBuildPartitionDesc(Relation rel, bool omit_detached)
 	Oid		   *oids = NULL;
 	bool	   *is_leaf = NULL;
 	bool		detached_exist;
+	bool		is_omit;
+	TransactionId detached_xmin;
 	ListCell   *cell;
 	int			i,
 				nparts;
 	PartitionKey key = RelationGetPartitionKey(rel);
+	MemoryContext *context;
 	MemoryContext new_pdcxt;
 	MemoryContext oldcxt;
 	int		   *mapping;
@@ -132,8 +159,11 @@ RelationBuildPartitionDesc(Relation rel, bool omit_detached)
 	 * some well-defined point in time.
 	 */
 	detached_exist = false;
-	inhoids = find_inheritance_children(RelationGetRelid(rel), omit_detached,
-										NoLock, &detached_exist);
+	detached_xmin = InvalidTransactionId;
+	inhoids = find_inheritance_children_extended(RelationGetRelid(rel),
+												 omit_detached, NoLock,
+												 &detached_exist,
+												 &detached_xmin);
 	nparts = list_length(inhoids);
 
 	/* Allocate working arrays for OIDs, leaf flags, and boundspecs. */
@@ -280,38 +310,39 @@ RelationBuildPartitionDesc(Relation rel, bool omit_detached)
 		MemoryContextSwitchTo(oldcxt);
 	}
 
+	is_omit = omit_detached && detached_exist && ActiveSnapshotSet();
+
 	/*
 	 * We have a fully valid partdesc.  Reparent it so that it has the right
-	 * lifespan, and if appropriate put it into the relation's relcache entry.
+	 * lifespan.
+	 */
+	MemoryContextSetParent(new_pdcxt, CacheMemoryContext);
+
+	/*
+	 * But first, a kluge: if there's an old context for this type of
+	 * descriptor, it contains an old partition descriptor that may still be
+	 * referenced somewhere.  Preserve it, while not leaking it, by
+	 * reattaching it as a child context of the new one.  Eventually it will
+	 * get dropped by either RelationClose or RelationClearRelation.
 	 */
-	if (omit_detached && detached_exist)
+	context = is_omit ? &rel->rd_pddcxt : &rel->rd_pdcxt;
+	if (*context != NULL)
+		MemoryContextSetParent(*context, new_pdcxt);
+	*context = new_pdcxt;
+
+	/*
+	 * Store it into relcache.  For partdescs built excluding detached
+	 * partitions, which we save separately, we also record the
+	 * pg_inherits.xmin of the detached partition that was omitted; this
+	 * informs a future potential user of such a cached partdescs.
+	 */
+	if (is_omit)
 	{
-		/*
-		 * A transient partition descriptor is only good for the current
-		 * statement, so make it a child of the current portal's context.
-		 */
-		MemoryContextSetParent(new_pdcxt, PortalContext);
+		rel->rd_partdesc_nodetached = partdesc;
+		rel->rd_partdesc_nodetached_xmin = detached_xmin;
 	}
 	else
 	{
-		/*
-		 * This partdesc goes into relcache.
-		 */
-
-		MemoryContextSetParent(new_pdcxt, CacheMemoryContext);
-
-		/*
-		 * But first, a kluge: if there's an old rd_pdcxt, it contains an old
-		 * partition descriptor that may still be referenced somewhere.
-		 * Preserve it, while not leaking it, by reattaching it as a child
-		 * context of the new rd_pdcxt.  Eventually it will get dropped by
-		 * either RelationClose or RelationClearRelation.
-		 */
-		if (rel->rd_pdcxt != NULL)
-			MemoryContextSetParent(rel->rd_pdcxt, new_pdcxt);
-		rel->rd_pdcxt = new_pdcxt;
-
-		/* Store it into relcache */
 		rel->rd_partdesc = partdesc;
 	}
 
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 466c28d528..31c8e07f2a 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -1157,7 +1157,10 @@ RelationBuildDesc(Oid targetRelId, bool insertIt)
 	relation->rd_partkey = NULL;
 	relation->rd_partkeycxt = NULL;
 	relation->rd_partdesc = NULL;
+	relation->rd_partdesc_nodetached = NULL;
+	relation->rd_partdesc_nodetached_xmin = InvalidTransactionId;
 	relation->rd_pdcxt = NULL;
+	relation->rd_pddcxt = NULL;
 	relation->rd_partcheck = NIL;
 	relation->rd_partcheckvalid = false;
 	relation->rd_partcheckcxt = NULL;
@@ -2103,10 +2106,16 @@ RelationClose(Relation relation)
 	 * stale partition descriptors it has.  This is unlikely, so check to see
 	 * if there are child contexts before expending a call to mcxt.c.
 	 */
-	if (RelationHasReferenceCountZero(relation) &&
-		relation->rd_pdcxt != NULL &&
-		relation->rd_pdcxt->firstchild != NULL)
-		MemoryContextDeleteChildren(relation->rd_pdcxt);
+	if (RelationHasReferenceCountZero(relation))
+	{
+		if (relation->rd_pdcxt != NULL &&
+			relation->rd_pdcxt->firstchild != NULL)
+			MemoryContextDeleteChildren(relation->rd_pdcxt);
+
+		if (relation->rd_pddcxt != NULL &&
+			relation->rd_pddcxt->firstchild != NULL)
+			MemoryContextDeleteChildren(relation->rd_pddcxt);
+	}
 
 #ifdef RELCACHE_FORCE_RELEASE
 	if (RelationHasReferenceCountZero(relation) &&
@@ -2390,6 +2399,8 @@ RelationDestroyRelation(Relation relation, bool remember_tupdesc)
 		MemoryContextDelete(relation->rd_partkeycxt);
 	if (relation->rd_pdcxt)
 		MemoryContextDelete(relation->rd_pdcxt);
+	if (relation->rd_pddcxt)
+		MemoryContextDelete(relation->rd_pddcxt);
 	if (relation->rd_partcheckcxt)
 		MemoryContextDelete(relation->rd_partcheckcxt);
 	pfree(relation);
@@ -2644,7 +2655,7 @@ RelationClearRelation(Relation relation, bool rebuild)
 			SWAPFIELD(PartitionKey, rd_partkey);
 			SWAPFIELD(MemoryContext, rd_partkeycxt);
 		}
-		if (newrel->rd_pdcxt != NULL)
+		if (newrel->rd_pdcxt != NULL || newrel->rd_pddcxt != NULL)
 		{
 			/*
 			 * We are rebuilding a partitioned relation with a non-zero
@@ -2672,13 +2683,22 @@ RelationClearRelation(Relation relation, bool rebuild)
 			 * newrel.
 			 */
 			relation->rd_partdesc = NULL;	/* ensure rd_partdesc is invalid */
+			relation->rd_partdesc_nodetached = NULL;
+			relation->rd_partdesc_nodetached_xmin = InvalidTransactionId;
 			if (relation->rd_pdcxt != NULL) /* probably never happens */
 				MemoryContextSetParent(newrel->rd_pdcxt, relation->rd_pdcxt);
 			else
 				relation->rd_pdcxt = newrel->rd_pdcxt;
+			if (relation->rd_pddcxt != NULL)
+				MemoryContextSetParent(newrel->rd_pddcxt, relation->rd_pddcxt);
+			else
+				relation->rd_pddcxt = newrel->rd_pddcxt;
 			/* drop newrel's pointers so we don't destroy it below */
 			newrel->rd_partdesc = NULL;
+			newrel->rd_partdesc_nodetached = NULL;
+			newrel->rd_partdesc_nodetached_xmin = InvalidTransactionId;
 			newrel->rd_pdcxt = NULL;
+			newrel->rd_pddcxt = NULL;
 		}
 
 #undef SWAPFIELD
@@ -6017,7 +6037,10 @@ load_relcache_init_file(bool shared)
 		rel->rd_partkey = NULL;
 		rel->rd_partkeycxt = NULL;
 		rel->rd_partdesc = NULL;
+		rel->rd_partdesc_nodetached = NULL;
+		rel->rd_partdesc_nodetached_xmin = InvalidTransactionId;
 		rel->rd_pdcxt = NULL;
+		rel->rd_pddcxt = NULL;
 		rel->rd_partcheck = NIL;
 		rel->rd_partcheckvalid = false;
 		rel->rd_partcheckcxt = NULL;
diff --git a/src/include/catalog/pg_inherits.h b/src/include/catalog/pg_inherits.h
index 4d28ede5a6..f47925588a 100644
--- a/src/include/catalog/pg_inherits.h
+++ b/src/include/catalog/pg_inherits.h
@@ -50,8 +50,10 @@ DECLARE_INDEX(pg_inherits_parent_index, 2187, on pg_inherits using btree(inhpare
 #define InheritsParentIndexId  2187
 
 
-extern List *find_inheritance_children(Oid parentrelId, bool omit_detached,
-									   LOCKMODE lockmode, bool *detached_exist);
+extern List *find_inheritance_children(Oid parentrelId, LOCKMODE lockmode);
+extern List *find_inheritance_children_extended(Oid parentrelId, bool omit_detached,
+									   LOCKMODE lockmode, bool *detached_exist, TransactionId *detached_xmin);
+
 extern List *find_all_inheritors(Oid parentrelId, LOCKMODE lockmode,
 								 List **parents);
 extern bool has_subclass(Oid relationId);
diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h
index 34e25eb597..cdc2560f94 100644
--- a/src/include/utils/rel.h
+++ b/src/include/utils/rel.h
@@ -129,6 +129,11 @@ typedef struct RelationData
 	PartitionDesc rd_partdesc;	/* partition descriptor, or NULL */
 	MemoryContext rd_pdcxt;		/* private context for rd_partdesc, if any */
 
+	/* Same as above, for partdescs that omit detached partitions */
+	PartitionDesc rd_partdesc_nodetached;	/* partdesc w/o detached parts */
+	TransactionId rd_partdesc_nodetached_xmin;	/* xmin for the above */
+	MemoryContext rd_pddcxt;	/* for rd_partdesc_nodetached, if any */
+
 	/* data managed by RelationGetPartitionQual: */
 	List	   *rd_partcheck;	/* partition CHECK quals */
 	bool		rd_partcheckvalid;	/* true if list has been computed */
diff --git a/src/test/isolation/expected/detach-partition-concurrently-3.out b/src/test/isolation/expected/detach-partition-concurrently-3.out
index 88e83638c7..dbb3825792 100644
--- a/src/test/isolation/expected/detach-partition-concurrently-3.out
+++ b/src/test/isolation/expected/detach-partition-concurrently-3.out
@@ -20,6 +20,7 @@ step s1describe: SELECT 'd3_listp' AS root, * FROM pg_partition_tree('d3_listp')
 root           relid          parentrelid    isleaf         level          
 
 d3_listp       d3_listp                      f              0              
+d3_listp       d3_listp2      d3_listp       t              1              
 d3_listp1      d3_listp1                     t              0              
 step s1alter: ALTER TABLE d3_listp1 ALTER a DROP NOT NULL;
 ERROR:  cannot alter partition "d3_listp1" with an incomplete detach
@@ -123,6 +124,61 @@ a
 
 1              
 
+starting permutation: s2snitch s1b s1s s2detach s1cancel s2detach2 s1c
+step s2snitch: INSERT INTO d3_pid SELECT pg_backend_pid();
+step s1b: BEGIN;
+step s1s: SELECT * FROM d3_listp;
+a              
+
+1              
+step s2detach: ALTER TABLE d3_listp DETACH PARTITION d3_listp1 CONCURRENTLY; <waiting ...>
+step s1cancel: SELECT pg_cancel_backend(pid) FROM d3_pid;
+pg_cancel_backend
+
+t              
+step s2detach: <... completed>
+error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2detach2: ALTER TABLE d3_listp DETACH PARTITION d3_listp2 CONCURRENTLY;
+ERROR:  partition "d3_listp1" already pending detach in partitioned table "public.d3_listp"
+step s1c: COMMIT;
+
+starting permutation: s2snitch s1b s1s s2detach s1cancel s2detachfinal s1c s2detach2
+step s2snitch: INSERT INTO d3_pid SELECT pg_backend_pid();
+step s1b: BEGIN;
+step s1s: SELECT * FROM d3_listp;
+a              
+
+1              
+step s2detach: ALTER TABLE d3_listp DETACH PARTITION d3_listp1 CONCURRENTLY; <waiting ...>
+step s1cancel: SELECT pg_cancel_backend(pid) FROM d3_pid;
+pg_cancel_backend
+
+t              
+step s2detach: <... completed>
+error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2detachfinal: ALTER TABLE d3_listp DETACH PARTITION d3_listp1 FINALIZE; <waiting ...>
+step s1c: COMMIT;
+step s2detachfinal: <... completed>
+step s2detach2: ALTER TABLE d3_listp DETACH PARTITION d3_listp2 CONCURRENTLY;
+
+starting permutation: s2snitch s1b s1s s2detach s1cancel s1c s1droppart s2detach2
+step s2snitch: INSERT INTO d3_pid SELECT pg_backend_pid();
+step s1b: BEGIN;
+step s1s: SELECT * FROM d3_listp;
+a              
+
+1              
+step s2detach: ALTER TABLE d3_listp DETACH PARTITION d3_listp1 CONCURRENTLY; <waiting ...>
+step s1cancel: SELECT pg_cancel_backend(pid) FROM d3_pid;
+pg_cancel_backend
+
+t              
+step s2detach: <... completed>
+error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s1c: COMMIT;
+step s1droppart: DROP TABLE d3_listp1;
+step s2detach2: ALTER TABLE d3_listp DETACH PARTITION d3_listp2 CONCURRENTLY;
+
 starting permutation: s2snitch s1b s1s s2detach s1cancel s1c s2begin s2drop s1s s2commit
 step s2snitch: INSERT INTO d3_pid SELECT pg_backend_pid();
 step s1b: BEGIN;
@@ -279,4 +335,3 @@ step s2detachfinal: ALTER TABLE d3_listp DETACH PARTITION d3_listp1 FINALIZE;
 step s1insertpart: INSERT INTO d3_listp1 VALUES (1); <waiting ...>
 step s2commit: COMMIT;
 step s1insertpart: <... completed>
-unused step name: s1droppart
diff --git a/src/test/isolation/specs/detach-partition-concurrently-3.spec b/src/test/isolation/specs/detach-partition-concurrently-3.spec
index 4b706430e1..4c563890eb 100644
--- a/src/test/isolation/specs/detach-partition-concurrently-3.spec
+++ b/src/test/isolation/specs/detach-partition-concurrently-3.spec
@@ -4,12 +4,13 @@ setup
 {
   CREATE TABLE d3_listp (a int) PARTITION BY LIST(a);
   CREATE TABLE d3_listp1 PARTITION OF d3_listp FOR VALUES IN (1);
+  CREATE TABLE d3_listp2 PARTITION OF d3_listp FOR VALUES IN (2);
   CREATE TABLE d3_pid (pid int);
   INSERT INTO d3_listp VALUES (1);
 }
 
 teardown {
-    DROP TABLE IF EXISTS d3_listp, d3_listp1, d3_pid;
+    DROP TABLE IF EXISTS d3_listp, d3_listp1, d3_listp2, d3_pid;
 }
 
 session "s1"
@@ -34,6 +35,7 @@ session "s2"
 step "s2begin"		{ BEGIN; }
 step "s2snitch"		{ INSERT INTO d3_pid SELECT pg_backend_pid(); }
 step "s2detach"		{ ALTER TABLE d3_listp DETACH PARTITION d3_listp1 CONCURRENTLY; }
+step "s2detach2"	{ ALTER TABLE d3_listp DETACH PARTITION d3_listp2 CONCURRENTLY; }
 step "s2detachfinal"	{ ALTER TABLE d3_listp DETACH PARTITION d3_listp1 FINALIZE; }
 step "s2drop"		{ DROP TABLE d3_listp1; }
 step "s2commit"		{ COMMIT; }
@@ -49,6 +51,11 @@ permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s1c" "s1drop" "s1list"
 # "truncate" only does parent, not partition
 permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s1c" "s1trunc" "s1spart"
 
+# If a partition pending detach exists, we cannot drop another one
+permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s2detach2" "s1c"
+permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s2detachfinal" "s1c" "s2detach2"
+permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s1c" "s1droppart" "s2detach2"
+
 # When a partition with incomplete detach is dropped, we grab lock on parent too.
 permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s1c" "s2begin" "s2drop" "s1s" "s2commit"
 
-- 
2.20.1


--n8g4imXOkfNTN/H1--





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ec86a17b3b3..64772877b0f 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 37b150b44ba..c99edae575b 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 8a560903ede..c1e9fe366f3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -402,6 +407,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v21-0001-Add-CopyToRountine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension
@ 2024-01-23 06:12  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
  method for CopyToState and it's used for flushing. End-of-row related
  codes were moved to CopyToTextSendEndOfRow().
---
 src/backend/commands/copyto.c  | 15 +++++++--------
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cfc74ee7b1..b5d8678394 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate);
 static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
 static void CopySendString(CopyToState cstate, const char *str);
 static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
 static void CopySendInt32(CopyToState cstate, int32 val);
 static void CopySendInt16(CopyToState cstate, int16 val);
 
@@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
 		default:
 			break;
 	}
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 static void
@@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
 		}
 	}
 
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 static void
@@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate)
 	/* Generate trailer for a binary copy */
 	CopySendInt16(cstate, -1);
 	/* Need to flush out the trailer */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 CopyToRoutine CopyToRoutineText = {
@@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate)
  * CopySendData sends output data to the destination (file or frontend)
  * CopySendString does the same for null-terminated strings
  * CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- *	(data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ *	(data is not actually flushed except by CopyToStateFlush)
  *
  * NB: no data conversion is applied by these functions
  *----------
@@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c)
 	appendStringInfoCharMacro(cstate->fe_msgbuf, c);
 }
 
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
 {
 	StringInfo	fe_msgbuf = cstate->fe_msgbuf;
 
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index a869d78d72..ffad433a21 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -174,6 +174,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.43.0


----Next_Part(Wed_Jan_24_14_49_36_2024_411)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v6-0005-Extract-COPY-FROM-format-implementations.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 74844103228..a115d7f9e26 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 54aa6cdecaf..b8d0e996117 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -500,6 +500,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index e298b19860c..5665408eaa0 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -402,6 +407,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Jul_30_16_13_06_2024_473)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
  method for CopyToState and it's used for flushing. End-of-row related
  codes were moved to CopyToTextSendEndOfRow().

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
  method for CopyFromState and "BinaryData" is redundant.
---
 src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
 src/backend/commands/copyto.c        | 15 +++++++--------
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 74844103228..cd80d34f3da 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -164,7 +164,6 @@ static int	CopyGetData(CopyFromState cstate, void *databuf,
 static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
 static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
 static void CopyLoadInputBuf(CopyFromState cstate);
-static int	CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
 
 void
 ReceiveCopyBegin(CopyFromState cstate)
@@ -193,7 +192,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	int32		tmp;
 
 	/* Signature */
-	if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+	if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
 		memcmp(readSig, BinarySignature, 11) != 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -221,7 +220,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	/* Skip extension header, if present */
 	while (tmp-- > 0)
 	{
-		if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+		if (CopyFromStateRead(cstate, readSig, 1) != 1)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("invalid COPY file header (wrong length)")));
@@ -363,7 +362,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
 {
 	uint32		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -380,7 +379,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
 {
 	uint16		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -691,14 +690,14 @@ CopyLoadInputBuf(CopyFromState cstate)
 }
 
 /*
- * CopyReadBinaryData
+ * CopyFromStateRead
  *
  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
  * and writes them to 'dest'.  Returns the number of bytes read (which
  * would be less than 'nbytes' only if we reach EOF).
  */
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
 {
 	int			copied_bytes = 0;
 
@@ -1078,7 +1077,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext,
 		 */
 		char		dummy;
 
-		if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+		if (CopyFromStateRead(cstate, &dummy, 1) > 0)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("received copy data after EOF marker")));
@@ -2103,8 +2102,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
 	resetStringInfo(&cstate->attribute_buf);
 
 	enlargeStringInfo(&cstate->attribute_buf, fld_size);
-	if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
-						   fld_size) != fld_size)
+	if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+						  fld_size) != fld_size)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 				 errmsg("unexpected EOF in COPY data")));
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 54aa6cdecaf..cd9e352533a 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -63,7 +63,6 @@ static void SendCopyEnd(CopyToState cstate);
 static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
 static void CopySendString(CopyToState cstate, const char *str);
 static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
 static void CopySendInt32(CopyToState cstate, int32 val);
 static void CopySendInt16(CopyToState cstate, int16 val);
 
@@ -99,7 +98,7 @@ CopyToTextLikeSendEndOfRow(CopyToState cstate)
 	}
 
 	/* Now take the actions related to the end of a row */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -325,7 +324,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
 		}
 	}
 
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -339,7 +338,7 @@ CopyToBinaryEnd(CopyToState cstate)
 	/* Generate trailer for a binary copy */
 	CopySendInt16(cstate, -1);
 	/* Need to flush out the trailer */
-	CopySendEndOfRow(cstate);
+	CopyToStateFlush(cstate);
 }
 
 /*
@@ -419,8 +418,8 @@ SendCopyEnd(CopyToState cstate)
  * CopySendData sends output data to the destination (file or frontend)
  * CopySendString does the same for null-terminated strings
  * CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- *	(data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ *	(data is not actually flushed except by CopyToStateFlush)
  *
  * NB: no data conversion is applied by these functions
  *----------
@@ -443,8 +442,8 @@ CopySendChar(CopyToState cstate, char c)
 	appendStringInfoCharMacro(cstate->fe_msgbuf, c);
 }
 
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
 {
 	StringInfo	fe_msgbuf = cstate->fe_msgbuf;
 
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 3104d99ea9f..0820b47a2d2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -401,6 +406,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Wed_Jul_24_17_30_59_2024_070)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension
@ 2024-01-23 06:12  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw)

For custom COPY TO format implementation:

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()

For custom COPY FROM format implementation:

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/backend/commands/copyto.c        | 14 ++++++++++++++
 src/include/commands/copyapi.h       | 10 ++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ccfbacb4a37..4fa23d992f5 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index fb68f42ce1e..93b041352c5 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 206d4c9fac9..2de610ef729 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -302,8 +302,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 
 typedef struct CopyToStateData *CopyToState;
 
@@ -405,6 +410,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v23-0001-Add-CopyToRountine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension
@ 2024-01-24 05:19  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-01-24 05:19 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
  method for CopyFromState and "BinaryData" is redundant.
---
 src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index a78a790060..f8a194635d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -165,7 +165,6 @@ static int	CopyGetData(CopyFromState cstate, void *databuf,
 static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
 static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
 static void CopyLoadInputBuf(CopyFromState cstate);
-static int	CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
 
 void
 ReceiveCopyBegin(CopyFromState cstate)
@@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	int32		tmp;
 
 	/* Signature */
-	if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+	if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
 		memcmp(readSig, BinarySignature, 11) != 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
 	/* Skip extension header, if present */
 	while (tmp-- > 0)
 	{
-		if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+		if (CopyFromStateRead(cstate, readSig, 1) != 1)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("invalid COPY file header (wrong length)")));
@@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
 {
 	uint32		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
 {
 	uint16		buf;
 
-	if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+	if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
 	{
 		*val = 0;				/* suppress compiler warning */
 		return false;
@@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate)
 }
 
 /*
- * CopyReadBinaryData
+ * CopyFromStateRead
  *
  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
  * and writes them to 'dest'.  Returns the number of bytes read (which
  * would be less than 'nbytes' only if we reach EOF).
  */
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
 {
 	int			copied_bytes = 0;
 
@@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 		 */
 		char		dummy;
 
-		if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+		if (CopyFromStateRead(cstate, &dummy, 1) > 0)
 			ereport(ERROR,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("received copy data after EOF marker")));
@@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
 	resetStringInfo(&cstate->attribute_buf);
 
 	enlargeStringInfo(&cstate->attribute_buf, fld_size);
-	if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
-						   fld_size) != fld_size)
+	if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+						  fld_size) != fld_size)
 		ereport(ERROR,
 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 				 errmsg("unexpected EOF in COPY data")));
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b7e8f627bf..22accc83ab 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -314,8 +314,13 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 /*
  * Represents the different dest cases we need to worry about at
  * the bottom level
-- 
2.43.0


----Next_Part(Wed_Jan_24_14_49_36_2024_411)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension
@ 2024-09-28 14:59  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 14 ++++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index fb68f42ce1e..93b041352c5 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b6ddb5f6216..310a37ba728 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -190,6 +190,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v23-0006-Add-CopyFromRoutine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension
@ 2024-09-28 14:59  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 14 ++++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 37b150b44ba..c99edae575b 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * CopyToStateFlush
+ *
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * These functions do apply some data conversion
  */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 03779c15f43..30765951e2e 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -187,6 +187,11 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v21-0006-Add-CopyFromRoutine.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension
@ 2024-09-28 15:32  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/include/commands/copyapi.h       |  6 ++++++
 2 files changed, 20 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ccfbacb4a37..4fa23d992f5 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 0274e3487c3..2de610ef729 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -302,8 +302,14 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
+
 typedef struct CopyToStateData *CopyToState;
 
 /*
-- 
2.45.2


----Next_Part(Tue_Nov__5_17_43_28_2024_751)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension
@ 2024-09-28 15:32  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 14 ++++++++++++++
 src/include/commands/copyapi.h       |  6 ++++++
 2 files changed, 20 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index ec86a17b3b3..64772877b0f 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * CopyFromStateRead
+ *
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c118558ee71..c1e9fe366f3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -299,8 +299,14 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
+
 typedef struct CopyToStateData *CopyToState;
 
 /*
-- 
2.45.2


----Next_Part(Sun_Sep_29_00_56_45_2024_080)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 96b5e144a1d..cb9bfa0053f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.45.2


----Next_Part(Wed_Nov_27_16_53_44_2024_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v27-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Wed_Mar_19_11_56_17_2025_532)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v37-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 91fa46ddf6f..da281f32950 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 5d071b378d6..f8167af4c79 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 2df53dda8a0..4b82372691e 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts);
-- 
2.47.1


----Next_Part(Fri_Jan_31_00_42_13_2025_303)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v29-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 17d89c23af0..35f9035141a 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Sat_Mar__1_11_50_09_2025_878)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v35-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Thu_Mar_20_10_24_55_2025_309)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v38-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 16d3b389e97..20d49d73e38 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c125dc3e209..d0da9e07a0d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.1


----Next_Part(Thu_Feb__6_21_06_31_2025_538)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v32-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 1e75e07dc0b..7487190bdd6 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.47.1


----Next_Part(Thu_Jan_23_18_12_10_2025_763)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v28-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 91fa46ddf6f..da281f32950 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 5d071b378d6..f8167af4c79 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 2df53dda8a0..4b82372691e 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts);
-- 
2.47.1


----Next_Part(Sat_Feb__1_08_10_23_2025_403)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v30-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 99c2f2dd699..f5ed3efbace 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 4f4ffabf882..5c5ea6592e3 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.2


----Next_Part(Wed_Mar__5_09_06_08_2025_350)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v36-0004-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c  | 12 ++++++++++++
 src/include/commands/copyapi.h |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 96b5e144a1d..cb9bfa0053f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1cb2815deab..030a82aca7f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -116,8 +116,13 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
-- 
2.45.2


----Next_Part(Mon_Nov_25_15_01_50_2024_156)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v26-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension
@ 2024-11-25 05:01  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw)

* Add CopyToStateData::opaque that can be used to keep data for custom
  COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
  as CopyToStateFlush()
---
 src/backend/commands/copyto.c          | 12 ++++++++++++
 src/include/commands/copyapi.h         |  2 ++
 src/include/commands/copyto_internal.h |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 16d3b389e97..20d49d73e38 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * the line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c125dc3e209..d0da9e07a0d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -54,6 +54,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation.	 Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 1b58b36c0a3..ce1c33a4004 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
-- 
2.47.1


----Next_Part(Sat_Feb__1_19_12_01_2025_760)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v31-0006-Add-support-for-adding-custom-COPY-FROM-format.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index f7982bf692f..650b6b2382b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index bf933069fea..d1a1dbeb178 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -105,4 +105,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3743b11faa4..a65bbbc962e 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Sat_Feb__1_08_10_23_2025_403)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v30-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 75b49629f08..01f2e7a8824 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 103eb21767d..ac58adbd23d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -104,4 +104,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Sat_Feb__1_19_12_01_2025_760)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v31-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 75b49629f08..01f2e7a8824 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 103eb21767d..ac58adbd23d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -104,4 +104,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Thu_Feb__6_21_06_31_2025_538)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v32-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Sat_Mar__1_11_50_09_2025_878)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v35-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Wed_Mar__5_09_06_08_2025_350)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v36-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Wed_Mar_19_11_56_17_2025_532)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v37-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..d8fd238e72b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 895c105d8d8..2044d8b8c4c 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -108,4 +108,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.2


----Next_Part(Thu_Mar_20_10_24_55_2025_309)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v38-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyGetData() to get the next data as
  CopyFromStateGetData()
---
 src/backend/commands/copyfromparse.c     | 11 +++++++++++
 src/include/commands/copyapi.h           |  2 ++
 src/include/commands/copyfrom_internal.h |  3 +++
 3 files changed, 16 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index f7982bf692f..650b6b2382b 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index bf933069fea..d1a1dbeb178 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -105,4 +105,6 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3743b11faa4..a65bbbc962e 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
-- 
2.47.1


----Next_Part(Fri_Jan_31_00_42_13_2025_303)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v29-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 1c68b0d2952..0a7e7255b7d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 335584f8877..caba308533d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Mon_Nov_25_15_01_50_2024_156)----





^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 1c68b0d2952..0a7e7255b7d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 335584f8877..caba308533d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.45.2


----Next_Part(Wed_Nov_27_16_53_44_2024_871)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v27-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension
@ 2024-11-25 05:21  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw)

* Add CopyFromStateData::opaque that can be used to keep data for
  custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data as
  CopyFromStateRead()
---
 src/backend/commands/copyfromparse.c | 12 ++++++++++++
 src/include/commands/copyapi.h       |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 5fcdbea2c2a..d79b6ebe8a3 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyReadBinaryData() for extensions. We want to keep
+ * CopyReadBinaryData() as a static function for
+ * optimization. CopyReadBinaryData() calls in this file may be optimized by
+ * a compiler.
+ */
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
+{
+	return CopyReadBinaryData(cstate, dest, nbytes);
+}
+
 /*
  * Read raw fields in the next line for COPY FROM in text or csv mode.
  * Return false if no more lines.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 9358515c6f6..6f158272829 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -335,6 +335,11 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
+extern int	CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
 #endif							/* COPYAPI_H */
-- 
2.47.1


----Next_Part(Thu_Jan_23_18_12_10_2025_763)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v28-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v39 3/5] Add support for implementing custom COPY handler as extension
@ 2025-03-27 02:24  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw)

* TO: Add CopyToStateData::opaque that can be used to keep
  data for custom COPY TO handler implementation
* TO: Export CopySendEndOfRow() to send end of row data as
  CopyToStateFlush()
* FROM: Add CopyFromStateData::opaque that can be used to
  keep data for custom COPY FROM handler implementation
* FROM: Export CopyGetData() to get the next data as
  CopyFromStateGetData()
* FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and
  "LOG_VERBOSITY verbose"

COPY FROM extensions must call CopyFromSkipErrorRow() when
CopyFromOneRow callback reports an error by
errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and
"LOG_VERBOSITY verbose" cases.
---
 src/backend/commands/copyfromparse.c          | 93 ++++++++++++-------
 src/backend/commands/copyto.c                 | 12 +++
 src/include/commands/copyapi.h                |  6 ++
 src/include/commands/copyfrom_internal.h      |  3 +
 src/include/commands/copyto_internal.h        |  3 +
 .../test_copy_format/expected/no_schema.out   | 47 ++++++++++
 .../test_copy_format/sql/no_schema.sql        | 24 +++++
 .../test_copy_format/test_copy_format.c       | 80 +++++++++++++++-
 8 files changed, 231 insertions(+), 37 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 17e51f02e04..2070f51a963 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
@@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 	return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
 }
 
+/*
+ * Call this when you report an error by errsave() in your CopyFromOneRow
+ * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases
+ * for you.
+ */
+void
+CopyFromSkipErrorRow(CopyFromState cstate)
+{
+	Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
+
+	cstate->num_errors++;
+
+	if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
+	{
+		/*
+		 * Since we emit line number and column info in the below notice
+		 * message, we suppress error context information other than the
+		 * relation name.
+		 */
+		Assert(!cstate->relname_only);
+		cstate->relname_only = true;
+
+		if (cstate->cur_attval)
+		{
+			char	   *attval;
+
+			attval = CopyLimitPrintoutLength(cstate->cur_attval);
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"",
+						   (unsigned long long) cstate->cur_lineno,
+						   cstate->cur_attname,
+						   attval));
+			pfree(attval);
+		}
+		else
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input",
+						   (unsigned long long) cstate->cur_lineno,
+						   cstate->cur_attname));
+
+		/* reset relname_only */
+		cstate->relname_only = false;
+	}
+}
+
 /*
  * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
  *
@@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
 										(Node *) cstate->escontext,
 										&values[m]))
 		{
-			Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
-
-			cstate->num_errors++;
-
-			if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
-			{
-				/*
-				 * Since we emit line number and column info in the below
-				 * notice message, we suppress error context information other
-				 * than the relation name.
-				 */
-				Assert(!cstate->relname_only);
-				cstate->relname_only = true;
-
-				if (cstate->cur_attval)
-				{
-					char	   *attval;
-
-					attval = CopyLimitPrintoutLength(cstate->cur_attval);
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"",
-								   (unsigned long long) cstate->cur_lineno,
-								   cstate->cur_attname,
-								   attval));
-					pfree(attval);
-				}
-				else
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input",
-								   (unsigned long long) cstate->cur_lineno,
-								   cstate->cur_attname));
-
-				/* reset relname_only */
-				cstate->relname_only = false;
-			}
-
+			CopyFromSkipErrorRow(cstate);
 			return true;
 		}
 
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b7ff6466ce3..23cbdad184c 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -456,6 +456,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 53ad3337f86..500ece7d5bb 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
@@ -106,4 +108,8 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
+extern void CopyFromSkipErrorRow(CopyFromState cstate);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 3a306e3286e..af425cf5fd9 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index 12c4a0f5979..14ee0f50588 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
diff --git a/src/test/modules/test_copy_format/expected/no_schema.out b/src/test/modules/test_copy_format/expected/no_schema.out
index d5903632b2e..05d160c1eae 100644
--- a/src/test/modules/test_copy_format/expected/no_schema.out
+++ b/src/test/modules/test_copy_format/expected/no_schema.out
@@ -1,6 +1,8 @@
 CREATE EXTENSION test_copy_format;
 CREATE TABLE public.test (a smallint, b integer, c bigint);
 INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=true
 NOTICE:  CopyFromInFunc: attribute: smallint
@@ -8,7 +10,50 @@ NOTICE:  CopyFromInFunc: attribute: integer
 NOTICE:  CopyFromInFunc: attribute: bigint
 NOTICE:  CopyFromStart: the number of attributes: 3
 NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  invalid value: "6"
+CONTEXT:  COPY test, line 2, column a: "6"
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
 NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  skipping row due to data type incompatibility at line 2 for column "a": "6"
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
+NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  too much lines: 3
+CONTEXT:  COPY test, line 3
 COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=false
 NOTICE:  CopyToOutFunc: attribute: smallint
@@ -18,6 +63,8 @@ NOTICE:  CopyToStart: the number of attributes: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToEnd
 DROP TABLE public.test;
 DROP EXTENSION test_copy_format;
diff --git a/src/test/modules/test_copy_format/sql/no_schema.sql b/src/test/modules/test_copy_format/sql/no_schema.sql
index 1e049f799f0..1901c4a9f43 100644
--- a/src/test/modules/test_copy_format/sql/no_schema.sql
+++ b/src/test/modules/test_copy_format/sql/no_schema.sql
@@ -1,7 +1,31 @@
 CREATE EXTENSION test_copy_format;
 CREATE TABLE public.test (a smallint, b integer, c bigint);
 INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+987
+654
+\.
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+321
 \.
 COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
 DROP TABLE public.test;
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 1d754201336..34ec693a7ec 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -14,6 +14,7 @@
 #include "postgres.h"
 
 #include "commands/copyapi.h"
+#include "commands/copyfrom_internal.h"
 #include "commands/defrem.h"
 #include "utils/builtins.h"
 
@@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
 static bool
 TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 {
+	int			n_attributes = list_length(cstate->attnumlist);
+	char	   *line;
+	int			line_size = n_attributes + 1;	/* +1 is for new line */
+	int			read_bytes;
+
 	ereport(NOTICE, (errmsg("CopyFromOneRow")));
-	return false;
+
+	cstate->cur_lineno++;
+	line = palloc(line_size);
+	read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size);
+	if (read_bytes == 0)
+		return false;
+	if (read_bytes != line_size)
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("one line must be %d bytes: %d",
+						line_size, read_bytes)));
+
+	if (cstate->cur_lineno == 1)
+	{
+		/* Success */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		ListCell   *cur;
+		int			i = 0;
+
+		foreach(cur, cstate->attnumlist)
+		{
+			int			attnum = lfirst_int(cur);
+			int			m = attnum - 1;
+			Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+			if (att->atttypid == INT2OID)
+			{
+				values[i] = Int16GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT4OID)
+			{
+				values[i] = Int32GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT8OID)
+			{
+				values[i] = Int64GetDatum(line[i] - '0');
+			}
+			nulls[i] = false;
+			i++;
+		}
+	}
+	else if (cstate->cur_lineno == 2)
+	{
+		/* Soft error */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		int			attnum = lfirst_int(list_head(cstate->attnumlist));
+		int			m = attnum - 1;
+		Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+		char		value[2];
+
+		cstate->cur_attname = NameStr(att->attname);
+		value[0] = line[0];
+		value[1] = '\0';
+		cstate->cur_attval = value;
+		errsave((Node *) cstate->escontext,
+				(
+				 errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("invalid value: \"%c\"", line[0])));
+		CopyFromSkipErrorRow(cstate);
+		cstate->cur_attname = NULL;
+		cstate->cur_attval = NULL;
+		return true;
+	}
+	else
+	{
+		/* Hard error */
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("too much lines: %llu",
+						(unsigned long long) cstate->cur_lineno)));
+	}
+
+	return true;
 }
 
 static void
-- 
2.47.2


----Next_Part(Thu_Mar_27_12_28_40_2025_510)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v39-0004-Use-copy-handlers-for-built-in-formats.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

* [PATCH v40 3/6] Add support for implementing custom COPY handler as extension
@ 2025-03-27 02:24  Sutou Kouhei <[email protected]>
  0 siblings, 0 replies; 35+ messages in thread

From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw)

* TO: Add CopyToStateData::opaque that can be used to keep
  data for custom COPY TO handler implementation
* TO: Export CopySendEndOfRow() to send end of row data as
  CopyToStateFlush()
* FROM: Add CopyFromStateData::opaque that can be used to
  keep data for custom COPY FROM handler implementation
* FROM: Export CopyGetData() to get the next data as
  CopyFromStateGetData()
* FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and
  "LOG_VERBOSITY verbose"

COPY FROM extensions must call CopyFromSkipErrorRow() when
CopyFromOneRow callback reports an error by
errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and
"LOG_VERBOSITY verbose" cases.
---
 src/backend/commands/copyfromparse.c          | 93 ++++++++++++-------
 src/backend/commands/copyto.c                 | 12 +++
 src/include/commands/copyapi.h                |  6 ++
 src/include/commands/copyfrom_internal.h      |  3 +
 src/include/commands/copyto_internal.h        |  3 +
 .../expected/test_copy_format.out             | 50 ++++++++++
 .../test_copy_format/sql/test_copy_format.sql | 35 +++++++
 .../test_copy_format/test_copy_format.c       | 80 +++++++++++++++-
 8 files changed, 245 insertions(+), 37 deletions(-)

diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 9f7171d1478..de68b53b000 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
 	return copied_bytes;
 }
 
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread)
+{
+	return CopyGetData(cstate, dest, minread, maxread);
+}
+
 /*
  * This function is exposed for use by extensions that read raw fields in the
  * next line. See NextCopyFromRawFieldsInternal() for details.
@@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
 	return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
 }
 
+/*
+ * Call this when you report an error by errsave() in your CopyFromOneRow
+ * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases
+ * for you.
+ */
+void
+CopyFromSkipErrorRow(CopyFromState cstate)
+{
+	Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
+
+	cstate->num_errors++;
+
+	if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
+	{
+		/*
+		 * Since we emit line number and column info in the below notice
+		 * message, we suppress error context information other than the
+		 * relation name.
+		 */
+		Assert(!cstate->relname_only);
+		cstate->relname_only = true;
+
+		if (cstate->cur_attval)
+		{
+			char	   *attval;
+
+			attval = CopyLimitPrintoutLength(cstate->cur_attval);
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
+						   cstate->cur_lineno,
+						   cstate->cur_attname,
+						   attval));
+			pfree(attval);
+		}
+		else
+			ereport(NOTICE,
+					errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
+						   cstate->cur_lineno,
+						   cstate->cur_attname));
+
+		/* reset relname_only */
+		cstate->relname_only = false;
+	}
+}
+
 /*
  * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
  *
@@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
 										(Node *) cstate->escontext,
 										&values[m]))
 		{
-			Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
-
-			cstate->num_errors++;
-
-			if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
-			{
-				/*
-				 * Since we emit line number and column info in the below
-				 * notice message, we suppress error context information other
-				 * than the relation name.
-				 */
-				Assert(!cstate->relname_only);
-				cstate->relname_only = true;
-
-				if (cstate->cur_attval)
-				{
-					char	   *attval;
-
-					attval = CopyLimitPrintoutLength(cstate->cur_attval);
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
-								   cstate->cur_lineno,
-								   cstate->cur_attname,
-								   attval));
-					pfree(attval);
-				}
-				else
-					ereport(NOTICE,
-							errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
-								   cstate->cur_lineno,
-								   cstate->cur_attname));
-
-				/* reset relname_only */
-				cstate->relname_only = false;
-			}
-
+			CopyFromSkipErrorRow(cstate);
 			return true;
 		}
 
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 265b847e255..d6fcfdfb9b1 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -454,6 +454,18 @@ CopySendEndOfRow(CopyToState cstate)
 	resetStringInfo(fe_msgbuf);
 }
 
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+	CopySendEndOfRow(cstate);
+}
+
 /*
  * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the
  * line termination and do common appropriate things for the end of row.
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 53ad3337f86..500ece7d5bb 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -56,6 +56,8 @@ typedef struct CopyToRoutine
 	void		(*CopyToEnd) (CopyToState cstate);
 } CopyToRoutine;
 
+extern void CopyToStateFlush(CopyToState cstate);
+
 /*
  * API structure for a COPY FROM format implementation. Note this must be
  * allocated in a server-lifetime manner, typically as a static const struct.
@@ -106,4 +108,8 @@ typedef struct CopyFromRoutine
 	void		(*CopyFromEnd) (CopyFromState cstate);
 } CopyFromRoutine;
 
+extern int	CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread);
+
+extern void CopyFromSkipErrorRow(CopyFromState cstate);
+
 #endif							/* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 24157e11a73..f9e27152313 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -181,6 +181,9 @@ typedef struct CopyFromStateData
 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
 
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyFromStateData;
 
 extern void ReceiveCopyBegin(CopyFromState cstate);
diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h
index da796131988..3bd9d702bf0 100644
--- a/src/include/commands/copyto_internal.h
+++ b/src/include/commands/copyto_internal.h
@@ -78,6 +78,9 @@ typedef struct CopyToStateData
 	FmgrInfo   *out_functions;	/* lookup info for output functions */
 	MemoryContext rowcontext;	/* per-row evaluation context */
 	uint64		bytes_processed;	/* number of bytes processed so far */
+
+	/* For custom format implementation */
+	void	   *opaque;			/* private space */
 } CopyToStateData;
 
 #endif							/* COPYTO_INTERNAL_H */
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
index 3916b766615..47a875f0ab1 100644
--- a/src/test/modules/test_copy_format/expected/test_copy_format.out
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -4,6 +4,8 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 -- schema.
 CREATE EXTENSION test_copy_format;
 -- We can find a custom COPY handler without schema.
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=true
 NOTICE:  CopyFromInFunc: attribute: smallint
@@ -11,7 +13,50 @@ NOTICE:  CopyFromInFunc: attribute: integer
 NOTICE:  CopyFromInFunc: attribute: bigint
 NOTICE:  CopyFromStart: the number of attributes: 3
 NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  invalid value: "6"
+CONTEXT:  COPY copy_data, line 2, column a: "6"
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
 NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  skipping row due to data type incompatibility at line 2 for column "a": "6"
+NOTICE:  CopyFromOneRow
+NOTICE:  1 row was skipped due to data type incompatibility
+NOTICE:  CopyFromEnd
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+NOTICE:  test_copy_format: is_from=true
+NOTICE:  CopyFromInFunc: attribute: smallint
+NOTICE:  CopyFromInFunc: attribute: integer
+NOTICE:  CopyFromInFunc: attribute: bigint
+NOTICE:  CopyFromStart: the number of attributes: 3
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+NOTICE:  CopyFromOneRow
+ERROR:  too much lines: 3
+CONTEXT:  COPY copy_data, line 3
 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format');
 NOTICE:  test_copy_format: is_from=false
 NOTICE:  CopyToOutFunc: attribute: smallint
@@ -21,7 +66,12 @@ NOTICE:  CopyToStart: the number of attributes: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
+NOTICE:  CopyToOneRow: the number of valid values: 3
 NOTICE:  CopyToEnd
+-- Reset data.
+TRUNCATE copy_data;
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 DROP EXTENSION test_copy_format;
 -- Install custom COPY handlers to a schema that isn't included in
 -- search_path.
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
index b262794f878..c7beb2fb8ae 100644
--- a/src/test/modules/test_copy_format/sql/test_copy_format.sql
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -4,10 +4,45 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
 -- No WITH SCHEMA. It installs custom COPY handlers to the current
 -- schema.
 CREATE EXTENSION test_copy_format;
+
 -- We can find a custom COPY handler without schema.
+
+-- 987 is accepted.
+-- 654 is a hard error because ON_ERROR is stop by default.
 COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format');
+987
+654
 \.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+\.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose);
+987
+654
+\.
+
+-- 987 is accepted.
+-- 654 is a soft error because ON_ERROR is ignore.
+-- 321 is a hard error.
+COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore);
+987
+654
+321
+\.
+
 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format');
+
+-- Reset data.
+TRUNCATE copy_data;
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+
 DROP EXTENSION test_copy_format;
 
 
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 1d754201336..34ec693a7ec 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -14,6 +14,7 @@
 #include "postgres.h"
 
 #include "commands/copyapi.h"
+#include "commands/copyfrom_internal.h"
 #include "commands/defrem.h"
 #include "utils/builtins.h"
 
@@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
 static bool
 TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 {
+	int			n_attributes = list_length(cstate->attnumlist);
+	char	   *line;
+	int			line_size = n_attributes + 1;	/* +1 is for new line */
+	int			read_bytes;
+
 	ereport(NOTICE, (errmsg("CopyFromOneRow")));
-	return false;
+
+	cstate->cur_lineno++;
+	line = palloc(line_size);
+	read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size);
+	if (read_bytes == 0)
+		return false;
+	if (read_bytes != line_size)
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("one line must be %d bytes: %d",
+						line_size, read_bytes)));
+
+	if (cstate->cur_lineno == 1)
+	{
+		/* Success */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		ListCell   *cur;
+		int			i = 0;
+
+		foreach(cur, cstate->attnumlist)
+		{
+			int			attnum = lfirst_int(cur);
+			int			m = attnum - 1;
+			Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+			if (att->atttypid == INT2OID)
+			{
+				values[i] = Int16GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT4OID)
+			{
+				values[i] = Int32GetDatum(line[i] - '0');
+			}
+			else if (att->atttypid == INT8OID)
+			{
+				values[i] = Int64GetDatum(line[i] - '0');
+			}
+			nulls[i] = false;
+			i++;
+		}
+	}
+	else if (cstate->cur_lineno == 2)
+	{
+		/* Soft error */
+		TupleDesc	tupDesc = RelationGetDescr(cstate->rel);
+		int			attnum = lfirst_int(list_head(cstate->attnumlist));
+		int			m = attnum - 1;
+		Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+		char		value[2];
+
+		cstate->cur_attname = NameStr(att->attname);
+		value[0] = line[0];
+		value[1] = '\0';
+		cstate->cur_attval = value;
+		errsave((Node *) cstate->escontext,
+				(
+				 errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("invalid value: \"%c\"", line[0])));
+		CopyFromSkipErrorRow(cstate);
+		cstate->cur_attname = NULL;
+		cstate->cur_attval = NULL;
+		return true;
+	}
+	else
+	{
+		/* Hard error */
+		ereport(ERROR,
+				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+				 errmsg("too much lines: %llu",
+						(unsigned long long) cstate->cur_lineno)));
+	}
+
+	return true;
 }
 
 static void
-- 
2.47.2


----Next_Part(Fri_Apr_25_21_45_34_2025_836)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v40-0004-Use-copy-handlers-for-built-in-formats.patch"



^ permalink  raw  reply  [nested|flat] 35+ messages in thread

end of thread, other threads:[~2025-03-27 02:24 UTC | newest]

Thread overview: 35+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-04-26 18:53 [PATCH v3] Allow a partdesc-omitting-partitions to be cached Alvaro Herrera <[email protected]>
2024-01-23 06:12 [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]>
2024-01-23 06:12 [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-01-24 05:19 [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-09-28 14:59 [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-09-28 14:59 [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-09-28 15:32 [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-09-28 15:32 [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:01 [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2024-11-25 05:21 [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]>
2025-03-27 02:24 [PATCH v39 3/5] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>
2025-03-27 02:24 [PATCH v40 3/6] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox