public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v2] Check partitions not in use
125+ messages / 10 participants
[nested] [flat]
* [PATCH v2] Check partitions not in use
@ 2019-07-22 15:09 Alvaro Herrera <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Alvaro Herrera @ 2019-07-22 15:09 UTC (permalink / raw)
---
src/backend/commands/tablecmds.c | 56 +++++++++++++++++++++++++-------
1 file changed, 45 insertions(+), 11 deletions(-)
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index f39f993e01..fd17aa4b5a 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -354,6 +354,7 @@ static void ATSimplePermissions(Relation rel, int allowed_targets);
static void ATWrongRelkindError(Relation rel, int allowed_targets);
static void ATSimpleRecursion(List **wqueue, Relation rel,
AlterTableCmd *cmd, bool recurse, LOCKMODE lockmode);
+static void ATCheckPartitionsNotInUse(Relation rel, LOCKMODE lockmode);
static void ATTypedTableRecursion(List **wqueue, Relation rel, AlterTableCmd *cmd,
LOCKMODE lockmode);
static List *find_typed_table_dependencies(Oid typeOid, const char *typeName,
@@ -3421,8 +3422,7 @@ CheckTableNotInUse(Relation rel, const char *stmt)
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
/* translator: first %s is a SQL command, eg ALTER TABLE */
- errmsg("cannot %s \"%s\" because "
- "it is being used by active queries in this session",
+ errmsg("cannot %s \"%s\" because it is being used by active queries in this session",
stmt, RelationGetRelationName(rel))));
if (rel->rd_rel->relkind != RELKIND_INDEX &&
@@ -3431,8 +3431,7 @@ CheckTableNotInUse(Relation rel, const char *stmt)
ereport(ERROR,
(errcode(ERRCODE_OBJECT_IN_USE),
/* translator: first %s is a SQL command, eg ALTER TABLE */
- errmsg("cannot %s \"%s\" because "
- "it has pending trigger events",
+ errmsg("cannot %s \"%s\" because it has pending trigger events",
stmt, RelationGetRelationName(rel))));
}
@@ -3910,16 +3909,19 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd,
break;
case AT_AddIdentity:
ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE);
+ /* This command never recurses */
pass = AT_PASS_ADD_CONSTR;
break;
- case AT_DropIdentity:
- ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE);
- pass = AT_PASS_DROP;
- break;
case AT_SetIdentity:
ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE);
+ /* This command never recurses */
pass = AT_PASS_COL_ATTRS;
break;
+ case AT_DropIdentity:
+ ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE);
+ /* This command never recurses */
+ pass = AT_PASS_DROP;
+ break;
case AT_DropNotNull: /* ALTER COLUMN DROP NOT NULL */
ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE);
ATPrepDropNotNull(rel, recurse, recursing);
@@ -3985,7 +3987,8 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd,
break;
case AT_DropConstraint: /* DROP CONSTRAINT */
ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE);
- /* Recursion occurs during execution phase */
+ ATCheckPartitionsNotInUse(rel, lockmode);
+ /* Other recursion occurs during execution phase */
/* No command-specific prep needed except saving recurse flag */
if (recurse)
cmd->subtype = AT_DropConstraintRecurse;
@@ -5224,8 +5227,9 @@ ATSimpleRecursion(List **wqueue, Relation rel,
AlterTableCmd *cmd, bool recurse, LOCKMODE lockmode)
{
/*
- * Propagate to children if desired. Only plain tables and foreign tables
- * have children, so no need to search for other relkinds.
+ * Propagate to children if desired. Only plain tables, foreign tables
+ * and partitioned tables have children, so no need to search for other
+ * relkinds.
*/
if (recurse &&
(rel->rd_rel->relkind == RELKIND_RELATION ||
@@ -5259,6 +5263,36 @@ ATSimpleRecursion(List **wqueue, Relation rel,
}
}
+/*
+ * Obtain list of partitions of the given table, locking them all at the given
+ * lockmode and ensuring that they all pass CheckTableNotInUse.
+ *
+ * This function is a no-op if the given relation is not a partitioned table;
+ * in particular, nothing is done if it's a legacy inheritance parent.
+ */
+static void
+ATCheckPartitionsNotInUse(Relation rel, LOCKMODE lockmode)
+{
+ if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ {
+ List *inh;
+ ListCell *cell;
+
+ inh = find_all_inheritors(RelationGetRelid(rel), lockmode, NULL);
+ /* first element is the parent rel; must ignore it */
+ for_each_cell(cell, inh, list_second_cell(inh))
+ {
+ Relation childrel;
+
+ /* find_all_inheritors already got lock */
+ childrel = table_open(lfirst_oid(cell), NoLock);
+ CheckTableNotInUse(childrel, "ALTER TABLE");
+ table_close(childrel, NoLock);
+ }
+ list_free(inh);
+ }
+}
+
/*
* ATTypedTableRecursion
*
--
2.17.1
--uAKRQypu60I7Lcqm--
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
@ 2024-01-24 05:49 Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-24 05:49 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
I've implemented custom COPY format feature based on the
current design discussion. See the attached patches for
details.
I also implemented a PoC COPY format handler for Apache
Arrow with this implementation and it worked.
https://github.com/kou/pg-copy-arrow
The patches implement not only custom COPY TO format feature
but also custom COPY FROM format feature.
0001-0004 is for COPY TO and 0005-0008 is for COPY FROM.
For COPY TO:
0001: This adds CopyToRoutine and use it for text/csv/binary
formats. No implementation change. This just move codes.
0002: This adds support for adding custom COPY TO format by
"CREATE FUNCTION ${FORMAT_NAME}". This uses the same
approach provided by Sawada-san[1] but this doesn't
introduce a wrapper CopyRoutine struct for
Copy{To,From}Routine. Because I noticed that a wrapper
CopyRoutine struct is needless. Copy handler can just return
CopyToRoutine or CopyFromRtouine because both of them have
NodeTag. We can distinct a returned struct by the NodeTag.
[1] https://www.postgresql.org/message-id/[email protected]...
0003: This exports CopyToStateData. No implementation change
except CopyDest enum values. I changed COPY_ prefix to
COPY_DEST_ to avoid name conflict with CopySource enum
values. This just moves codes.
0004: This adds CopyToState::opaque and exports
CopySendEndOfRow(). CopySendEndOfRow() is renamed to
CopyToStateFlush().
For COPY FROM:
0005: Same as 0001 but for COPY FROM. This adds
CopyFromRoutine and use it for text/csv/binary formats. No
implementation change. This just move codes.
0006: Same as 0002 but for COPY FROM. This adds support for
adding custom COPY FROM format by "CREATE FUNCTION
${FORMAT_NAME}".
0007: Same as 0003 but for COPY FROM. This exports
CopyFromStateData. No implementation change except
CopySource enum values. I changed COPY_ prefix to
COPY_SOURCE_ to align CopyDest changes in 0003. This just
moves codes.
0008: Same as 0004 but for COPY FROM. This adds
CopyFromState::opaque and exports
CopyReadBinaryData(). CopyReadBinaryData() is renamed to
CopyFromStateRead().
Thanks,
--
kou
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Jan 2024 15:27:02 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
> Hi,
>
> If there are no more comments for the current design, I'll
> start implementing this feature with the following
> approaches for "Discussing" items:
>
>> 3.1 Should we use one function(internal) for COPY TO/FROM
>> handlers or two function(internal)s (one is for COPY TO
>> handler and another is for COPY FROM handler)?
>> [4]
>
> I'll choose "one function(internal) for COPY TO/FROM handlers".
>
>> 3.4 Should we export Copy{To,From}State? Or should we just
>> provide getters/setters to access Copy{To,From}State
>> internal?
>> [10]
>
> I'll export Copy{To,From}State.
>
>
> Thanks,
> --
> kou
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST),
> Sutou Kouhei <[email protected]> wrote:
>
>> Hi,
>>
>> Here is the current summary for a this discussion to make
>> COPY format extendable. It's for reaching consensus and
>> starting implementing the feature. (I'll start implementing
>> the feature once we reach consensus.) If you have any
>> opinion, please share it.
>>
>> Confirmed:
>>
>> 1.1 Making COPY format extendable will not reduce performance.
>> [1]
>>
>> Decisions:
>>
>> 2.1 Use separated handler for COPY TO and COPY FROM because
>> our COPY TO implementation (copyto.c) and COPY FROM
>> implementation (coypfrom.c) are separated.
>> [2]
>>
>> 2.2 Don't use system catalog for COPY TO/FROM handlers. We can
>> just use a function(internal) that returns a handler instead.
>> [3]
>>
>> 2.3 The implementation must include documentation.
>> [5]
>>
>> 2.4 The implementation must include test.
>> [6]
>>
>> 2.5 The implementation should be consist of small patches
>> for easy to review.
>> [6]
>>
>> 2.7 Copy{To,From}State must have a opaque space for
>> handlers.
>> [8]
>>
>> 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO
>> handlers.
>> [8]
>>
>> 2.9 Make "format" in PgMsg_CopyOutResponse message
>> extendable.
>> [9]
>>
>> 2.10 Make makeNode() call avoidable in function(internal)
>> that returns COPY TO/FROM handler.
>> [9]
>>
>> 2.11 Custom COPY TO/FROM handlers must be able to parse
>> their options.
>> [11]
>>
>> Discussing:
>>
>> 3.1 Should we use one function(internal) for COPY TO/FROM
>> handlers or two function(internal)s (one is for COPY TO
>> handler and another is for COPY FROM handler)?
>> [4]
>>
>> 3.2 If we use separated function(internal) for COPY TO/FROM
>> handlers, we need to define naming rule. For example,
>> <method_name>_to(internal) for COPY TO handler and
>> <method_name>_from(internal) for COPY FROM handler.
>> [4]
>>
>> 3.3 Should we use prefix or suffix for function(internal)
>> name to avoid name conflict with other handlers such as
>> tablesample handlers?
>> [7]
>>
>> 3.4 Should we export Copy{To,From}State? Or should we just
>> provide getters/setters to access Copy{To,From}State
>> internal?
>> [10]
>>
>>
>> [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com
>> [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz
>> [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m...
>> [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m...
>> [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp...
>> [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz
>> [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com
>> [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com
>> [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz
>> [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m...
>> [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com
>>
>>
>> Thanks,
>> --
>> kou
>>
>>
>
>
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-24 08:11 ` Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-24 08:11 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
> For COPY TO:
>
> 0001: This adds CopyToRoutine and use it for text/csv/binary
> formats. No implementation change. This just move codes.
10M without this change:
format,elapsed time (ms)
text,1090.763
csv,1136.103
binary,1137.141
10M with this change:
format,elapsed time (ms)
text,1082.654
csv,1196.991
binary,1069.697
These numbers point out that binary is faster by 6%, csv is slower by
5%, while text stays around what looks like noise range. That's not
negligible. Are these numbers reproducible? If they are, that could
be a problem for anybody doing bulk-loading of large data sets. I am
not sure to understand where the improvement for binary comes from by
reading the patch, but perhaps perf would tell more for each format?
The loss with csv could be blamed on the extra manipulations of the
function pointers, likely.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-24 12:15 ` Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Andrew Dunstan @ 2024-01-24 12:15 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers
On 2024-01-24 We 03:11, Michael Paquier wrote:
> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
>> For COPY TO:
>>
>> 0001: This adds CopyToRoutine and use it for text/csv/binary
>> formats. No implementation change. This just move codes.
> 10M without this change:
>
> format,elapsed time (ms)
> text,1090.763
> csv,1136.103
> binary,1137.141
>
> 10M with this change:
>
> format,elapsed time (ms)
> text,1082.654
> csv,1196.991
> binary,1069.697
>
> These numbers point out that binary is faster by 6%, csv is slower by
> 5%, while text stays around what looks like noise range. That's not
> negligible. Are these numbers reproducible? If they are, that could
> be a problem for anybody doing bulk-loading of large data sets. I am
> not sure to understand where the improvement for binary comes from by
> reading the patch, but perhaps perf would tell more for each format?
> The loss with csv could be blamed on the extra manipulations of the
> function pointers, likely.
I don't think that's at all acceptable.
We've spent quite a lot of blood sweat and tears over the years to make
COPY fast, and we should not sacrifice any of that lightly.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
@ 2024-01-24 14:17 ` Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 3 replies; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-24 14:17 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
Andrew Dunstan <[email protected]> wrote:
>
> On 2024-01-24 We 03:11, Michael Paquier wrote:
>> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
>>> For COPY TO:
>>>
>>> 0001: This adds CopyToRoutine and use it for text/csv/binary
>>> formats. No implementation change. This just move codes.
>> 10M without this change:
>>
>> format,elapsed time (ms)
>> text,1090.763
>> csv,1136.103
>> binary,1137.141
>>
>> 10M with this change:
>>
>> format,elapsed time (ms)
>> text,1082.654
>> csv,1196.991
>> binary,1069.697
>>
>> These numbers point out that binary is faster by 6%, csv is slower by
>> 5%, while text stays around what looks like noise range. That's not
>> negligible. Are these numbers reproducible? If they are, that could
>> be a problem for anybody doing bulk-loading of large data sets. I am
>> not sure to understand where the improvement for binary comes from by
>> reading the patch, but perhaps perf would tell more for each format?
>> The loss with csv could be blamed on the extra manipulations of the
>> function pointers, likely.
>
>
> I don't think that's at all acceptable.
>
> We've spent quite a lot of blood sweat and tears over the years to make COPY
> fast, and we should not sacrifice any of that lightly.
These numbers aren't reproducible. Because these benchmarks
executed on my normal machine not a machine only for
benchmarking. The machine runs another processes such as
editor and Web browser.
For example, here are some results with master
(94edfe250c6a200d2067b0debfe00b4122e9b11e):
Format,N records,Elapsed time (ms)
csv,10000000,1073.715
csv,10000000,1022.830
csv,10000000,1073.584
csv,10000000,1090.651
csv,10000000,1052.259
Here are some results with master + the 0001 patch:
Format,N records,Elapsed time (ms)
csv,10000000,1025.356
csv,10000000,1067.202
csv,10000000,1014.563
csv,10000000,1032.088
csv,10000000,1058.110
I uploaded my benchmark script so that you can run the same
benchmark on your machine:
https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
Could anyone try the benchmark with master and master+0001?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 02:53 ` jian he <[email protected]>
2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 2 replies; 125+ messages in thread
From: jian he @ 2024-01-25 02:53 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 10:17 PM Sutou Kouhei <[email protected]> wrote:
>
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
>
> Could anyone try the benchmark with master and master+0001?
>
sorry. I made a mistake. I applied v6, 0001 to 0008 all the patches.
my tests:
CREATE unlogged TABLE data (a bigint);
SELECT setseed(0.29);
INSERT INTO data SELECT random() * 10000 FROM generate_series(1, 1e7);
my setup:
meson setup --reconfigure ${BUILD} \
-Dprefix=${PG_PREFIX} \
-Dpgport=5462 \
-Dbuildtype=release \
-Ddocs_html_style=website \
-Ddocs_pdf=disabled \
-Dllvm=disabled \
-Dextra_version=_release_build
gcc version: PostgreSQL 17devel_release_build on x86_64-linux,
compiled by gcc-11.4.0, 64-bit
apply your patch:
COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Time: 668.996 ms
Time: 596.254 ms
Time: 592.723 ms
Time: 591.663 ms
Time: 590.803 ms
not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda
COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Time: 644.246 ms
Time: 583.075 ms
Time: 568.670 ms
Time: 569.463 ms
Time: 569.201 ms
I forgot to test other formats.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
@ 2024-01-25 03:28 ` Michael Paquier <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Michael Paquier @ 2024-01-25 03:28 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 10:53:58AM +0800, jian he wrote:
> apply your patch:
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
> Time: 668.996 ms
> Time: 596.254 ms
> Time: 592.723 ms
> Time: 591.663 ms
> Time: 590.803 ms
>
> not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
> Time: 644.246 ms
> Time: 583.075 ms
> Time: 568.670 ms
> Time: 569.463 ms
> Time: 569.201 ms
>
> I forgot to test other formats.
There can be some variance in the tests, so you'd better run much more
tests so as you can get a better idea of the mean. Discarding the N
highest and lowest values also reduces slightly the effects of the
noise you would get across single runs.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
@ 2024-01-25 08:05 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:05 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
Thanks for trying these patches!
In <CACJufxF9NS3xQ2d79jN0V1CGvF7cR16uJo-C3nrY7vZrwvxF7w@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 10:53:58 +0800,
jian he <[email protected]> wrote:
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Wow! I didn't know the "\watch count="!
I'll use it.
> Time: 668.996 ms
> Time: 596.254 ms
> Time: 592.723 ms
> Time: 591.663 ms
> Time: 590.803 ms
It seems that 5 times isn't enough for this case as Michael
said. But thanks for trying!
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 03:17 ` Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-25 03:17 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 11:17:26PM +0900, Sutou Kouhei wrote:
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
> Andrew Dunstan <[email protected]> wrote:
>> We've spent quite a lot of blood sweat and tears over the years to make COPY
>> fast, and we should not sacrifice any of that lightly.
Clearly.
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
Thanks, that saves time. I am attaching it to this email as well, for
the sake of the archives if this link is removed in the future.
> Could anyone try the benchmark with master and master+0001?
Yep. It is one point we need to settle before deciding what to do
with this patch set, and I've done so to reach my own conclusion.
I have a rather good machine at my disposal in the cloud, so I did a
few runs with HEAD and HEAD+0001, with PGDATA mounted on a tmpfs.
Here are some results for the 10M row case, as these should be the
least prone to noise, 5 runs each:
master
text 10M 1732.570 1684.542 1693.430 1687.696 1714.845
csv 10M 1729.113 1724.926 1727.414 1726.237 1728.865
bin 10M 1679.097 1677.887 1676.764 1677.554 1678.120
master+0001
text 10M 1702.207 1654.818 1647.069 1690.568 1654.446
csv 10M 1764.939 1714.313 1712.444 1712.323 1716.952
bin 10M 1703.061 1702.719 1702.234 1703.346 1704.137
Hmm. The point of contention in the patch is the change to use the
CopyToOneRow callback in CopyOneRowTo(), as we go through it for each
row and we should habe a worst-case scenario with a relation that has
a small attribute size. The more rows, the more effect it would have.
The memory context switches and the StringInfo manipulations are
equally important, and there are a bunch of the latter, actually, with
optimizations around fe_msgbuf.
I've repeated a few runs across these two builds, and there is some
variance and noise, but I am going to agree with your point that the
effect 0001 cannot be seen. Even HEAD is showing some noise. So I am
discarding the concerns I had after seeing the numbers you posted
upthread.
+typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
+typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
+typedef void (*CopyToEnd_function) (CopyToState cstate);
We don't really need a set of typedefs here, let's put the definitions
in the CopyToRoutine struct instead.
+extern CopyToRoutine CopyToRoutineText;
+extern CopyToRoutine CopyToRoutineCSV;
+extern CopyToRoutine CopyToRoutineBinary;
All that should IMO remain in copyto.c and copyfrom.c in the initial
patch doing the refactoring. Why not using a fetch function instead
that uses a string in input? Then you can call that once after
parsing the List of options in ProcessCopyOptions().
Introducing copyapi.h in the initial patch makes sense here for the TO
and FROM routines.
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
Some areas, like this comment, are written in an incorrect format.
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, colname);
You are right that this is not worth the trouble of creating a
different set of callbacks for CSV. This makes the result cleaner.
+ getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
Actually, this split is interesting. It is possible for a custom
format to plug in a custom set of out functions. Did you make use of
something custom for your own stuff? Actually, could it make sense to
split the assignment of cstate->out_functions into its own callback?
Sure, that's part of the start phase, but at least it would make clear
that a custom method *has* to assign these OIDs to work. The patch
implies that as a rule, without a comment that CopyToStart *must* set
up these OIDs.
I think that 0001 and 0005 should be handled first, as pieces
independent of the rest. Then we could move on with 0002~0004 and
0006~0008.
--
Michael
#!/bin/bash
set -eu
set -o pipefail
base_dir=$(dirname "$0")
prepare_sql()
{
size=$1
db_name=$2
cat <<SQL
DROP DATABASE IF EXISTS ${db_name};
CREATE DATABASE ${db_name};
\\c ${db_name}
CREATE TABLE data (int32 integer);
SELECT setseed(0.29);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${size});
SQL
}
measure()
{
local format=$1
local n_tries=6
for i in $(seq ${n_tries}); do
LANG=C \
PAGER=cat \
psql \
--no-psqlrc \
--command "\\timing" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \
grep --text -o '^Time: [0-9.]* ms' | \
grep -o '[0-9.]*'
done | sort --numeric-sort | head -n 9 | tail -n 1
}
result_suffix=""
if [ $# -gt 0 ]; then
result_suffix="-$1"
fi
result="${base_dir}/result${result_suffix}.csv"
echo "Format,N records,Elapsed time (ms)" | \
tee "${result}"
sizes=()
sizes+=(100000)
sizes+=(1000000)
sizes+=(10000000)
for size in "${sizes[@]}"; do
export PGDATABASE="copy_${size}"
echo "${size}: preparing"
prepare_sql "${size}" "${PGDATABASE}" | \
psql -d postgres
echo "${size}: text"
elapsed_time=$(measure text)
echo "text,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: csv"
elapsed_time=$(measure csv)
echo "csv,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: binary"
elapsed_time=$(measure binary)
echo "binary,${size},${elapsed_time}" | \
tee -a "${result}"
done
Attachments:
[text/plain] bench-run.txt (1.6K, ../../[email protected]/2-bench-run.txt)
download | inline:
#!/bin/bash
set -eu
set -o pipefail
base_dir=$(dirname "$0")
prepare_sql()
{
size=$1
db_name=$2
cat <<SQL
DROP DATABASE IF EXISTS ${db_name};
CREATE DATABASE ${db_name};
\\c ${db_name}
CREATE TABLE data (int32 integer);
SELECT setseed(0.29);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${size});
SQL
}
measure()
{
local format=$1
local n_tries=6
for i in $(seq ${n_tries}); do
LANG=C \
PAGER=cat \
psql \
--no-psqlrc \
--command "\\timing" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \
grep --text -o '^Time: [0-9.]* ms' | \
grep -o '[0-9.]*'
done | sort --numeric-sort | head -n 9 | tail -n 1
}
result_suffix=""
if [ $# -gt 0 ]; then
result_suffix="-$1"
fi
result="${base_dir}/result${result_suffix}.csv"
echo "Format,N records,Elapsed time (ms)" | \
tee "${result}"
sizes=()
sizes+=(100000)
sizes+=(1000000)
sizes+=(10000000)
for size in "${sizes[@]}"; do
export PGDATABASE="copy_${size}"
echo "${size}: preparing"
prepare_sql "${size}" "${PGDATABASE}" | \
psql -d postgres
echo "${size}: text"
elapsed_time=$(measure text)
echo "text,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: csv"
elapsed_time=$(measure csv)
echo "csv,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: binary"
elapsed_time=$(measure binary)
echo "binary,${size},${elapsed_time}" | \
tee -a "${result}"
done
[application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-25 08:45 ` Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900,
Michael Paquier <[email protected]> wrote:
> +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
> +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
> +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
> +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
> +typedef void (*CopyToEnd_function) (CopyToState cstate);
>
> We don't really need a set of typedefs here, let's put the definitions
> in the CopyToRoutine struct instead.
OK. I'll do it.
> +extern CopyToRoutine CopyToRoutineText;
> +extern CopyToRoutine CopyToRoutineCSV;
> +extern CopyToRoutine CopyToRoutineBinary;
>
> All that should IMO remain in copyto.c and copyfrom.c in the initial
> patch doing the refactoring. Why not using a fetch function instead
> that uses a string in input? Then you can call that once after
> parsing the List of options in ProcessCopyOptions().
OK. How about the following for the fetch function
signature?
extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
We may introduce an enum and use it:
typedef enum CopyBuiltinFormat
{
COPY_BUILTIN_FORMAT_TEXT = 0,
COPY_BUILTIN_FORMAT_CSV,
COPY_BUILTIN_FORMAT_BINARY,
} CopyBuiltinFormat;
extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format);
> +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
> + * move the code to here later. */
> Some areas, like this comment, are written in an incorrect format.
Oh, sorry. I assumed that the comment style was adjusted by
pgindent.
I'll use the following style:
/*
* ...
*/
> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>
> Actually, this split is interesting. It is possible for a custom
> format to plug in a custom set of out functions. Did you make use of
> something custom for your own stuff?
I didn't. My PoC custom COPY format handler for Apache Arrow
just handles integer and text for now. It doesn't use
cstate->out_functions because cstate->out_functions may not
return a valid binary format value for Apache Arrow. So it
formats each value by itself.
I'll chose one of them for a custom type (that isn't
supported by Apache Arrow, e.g. PostGIS types):
1. Report an unsupported error
2. Call output function for Apache Arrow provided by the
custom type
> Actually, could it make sense to
> split the assignment of cstate->out_functions into its own callback?
Yes. Because we need to use getTypeBinaryOutputInfo() for
"binary" and use getTypeOutputInfo() for "text" and "csv".
> Sure, that's part of the start phase, but at least it would make clear
> that a custom method *has* to assign these OIDs to work. The patch
> implies that as a rule, without a comment that CopyToStart *must* set
> up these OIDs.
CopyToStart doesn't need to set up them if the handler
doesn't use cstate->out_functions.
> I think that 0001 and 0005 should be handled first, as pieces
> independent of the rest. Then we could move on with 0002~0004 and
> 0006~0008.
OK. I'll focus on 0001 and 0005 for now. I'll restart
0002-0004/0006-0008 after 0001 and 0005 are accepted.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 23:35 ` Michael Paquier <[email protected]>
2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-25 23:35 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 05:45:43PM +0900, Sutou Kouhei wrote:
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900,
> Michael Paquier <[email protected]> wrote:
>> +extern CopyToRoutine CopyToRoutineText;
>> +extern CopyToRoutine CopyToRoutineCSV;
>> +extern CopyToRoutine CopyToRoutineBinary;
>>
>> All that should IMO remain in copyto.c and copyfrom.c in the initial
>> patch doing the refactoring. Why not using a fetch function instead
>> that uses a string in input? Then you can call that once after
>> parsing the List of options in ProcessCopyOptions().
>
> OK. How about the following for the fetch function
> signature?
>
> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad
history with naming things around here.
> We may introduce an enum and use it:
>
> typedef enum CopyBuiltinFormat
> {
> COPY_BUILTIN_FORMAT_TEXT = 0,
> COPY_BUILTIN_FORMAT_CSV,
> COPY_BUILTIN_FORMAT_BINARY,
> } CopyBuiltinFormat;
>
> extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format);
I am not sure that this is necessary as the option value is a string.
> Oh, sorry. I assumed that the comment style was adjusted by
> pgindent.
No worries, that's just something we get used to. I tend to fix a lot
of these things by myself when editing patches.
>> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
>> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>>
>> Actually, this split is interesting. It is possible for a custom
>> format to plug in a custom set of out functions. Did you make use of
>> something custom for your own stuff?
>
> I didn't. My PoC custom COPY format handler for Apache Arrow
> just handles integer and text for now. It doesn't use
> cstate->out_functions because cstate->out_functions may not
> return a valid binary format value for Apache Arrow. So it
> formats each value by itself.
I mean, if you use a custom output function, you could tweak things
even more with byteas or such.. If a callback is expected to do
something, like setting the output function OIDs in the start
callback, we'd better document it rather than letting that be implied.
>> Actually, could it make sense to
>> split the assignment of cstate->out_functions into its own callback?
>
> Yes. Because we need to use getTypeBinaryOutputInfo() for
> "binary" and use getTypeOutputInfo() for "text" and "csv".
Okay. After sleeping on it, a split makes sense here, because it also
reduces the presence of TupleDesc in the start callback.
>> Sure, that's part of the start phase, but at least it would make clear
>> that a custom method *has* to assign these OIDs to work. The patch
>> implies that as a rule, without a comment that CopyToStart *must* set
>> up these OIDs.
>
> CopyToStart doesn't need to set up them if the handler
> doesn't use cstate->out_functions.
Noted.
>> I think that 0001 and 0005 should be handled first, as pieces
>> independent of the rest. Then we could move on with 0002~0004 and
>> 0006~0008.
>
> OK. I'll focus on 0001 and 0005 for now. I'll restart
> 0002-0004/0006-0008 after 0001 and 0005 are accepted.
Once you get these, I'd be interested in re-doing an evaluation of
COPY TO and more tests with COPY FROM while running Postgres on
scissors. One thing I was thinking to use here is my blackhole_am for
COPY FROM:
https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am
As per its name, it does nothing on INSERT, so you could create a
table using it as access method, and stress the COPY FROM execution
paths without having to mount Postgres on a tmpfs because the data is
sent to the void. Perhaps it does not matter, but that moves the
tests to the bottlenecks we want to stress (aka the per-row callback
for large data sets).
I've switched the patch as waiting on author for now. Thanks for your
perseverance here. I understand that's not easy to follow up with
patches and reviews (^_^;)
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-26 08:49 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:49 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 08:35:19 +0900,
Michael Paquier <[email protected]> wrote:
>> OK. How about the following for the fetch function
>> signature?
>>
>> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
>
> Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad
> history with naming things around here.
Thanks for the suggestion.
I rethink about this and use the following:
+extern void ProcessCopyOptionFormatTo(ParseState *pstate, CopyFormatOptions *opts_out, DefElem *defel);
It's not a fetch function. It sets CopyToRoutine opts_out
instead. But it hides CopyToRoutine* to copyto.c. Is it
acceptable?
>> OK. I'll focus on 0001 and 0005 for now. I'll restart
>> 0002-0004/0006-0008 after 0001 and 0005 are accepted.
>
> Once you get these, I'd be interested in re-doing an evaluation of
> COPY TO and more tests with COPY FROM while running Postgres on
> scissors. One thing I was thinking to use here is my blackhole_am for
> COPY FROM:
> https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am
Thanks!
Could you evaluate the attached patch set with COPY FROM?
I attach v7 patch set. It includes only the 0001 and 0005
parts in v6 patch set because we focus on them for now.
0001: This is based on 0001 in v6.
Changes since v6:
* Fix comment style
* Hide CopyToRoutine{Text,CSV,Binary}
* Add more comments
* Eliminate "if (cstate->opts.csv_mode)" branches from "text"
and "csv" callbacks
* Remove CopyTo*_function typedefs
* Update benchmark results in commit message but the results
are measured on my environment that isn't suitable for
accurate benchmark
0002: This is based on 0005 in v6.
Changes since v6:
* Fix comment style
* Hide CopyFromRoutine{Text,CSV,Binary}
* Add more comments
* Eliminate a "if (cstate->opts.csv_mode)" branch from "text"
and "csv" callbacks
* NOTE: We can eliminate more "if (cstate->opts.csv_mode)" branches
such as one in NextCopyFromRawFields(). Should we do it
in this feature improvement (make COPY format
extendable)? Can we defer this as a separated improvement?
* Remove CopyFrom*_function typedefs
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 04:36 ` Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 2 replies; 125+ messages in thread
From: Masahiko Sawada @ 2024-01-25 04:36 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 11:17 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
> Andrew Dunstan <[email protected]> wrote:
>
> >
> > On 2024-01-24 We 03:11, Michael Paquier wrote:
> >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
> >>> For COPY TO:
> >>>
> >>> 0001: This adds CopyToRoutine and use it for text/csv/binary
> >>> formats. No implementation change. This just move codes.
> >> 10M without this change:
> >>
> >> format,elapsed time (ms)
> >> text,1090.763
> >> csv,1136.103
> >> binary,1137.141
> >>
> >> 10M with this change:
> >>
> >> format,elapsed time (ms)
> >> text,1082.654
> >> csv,1196.991
> >> binary,1069.697
> >>
> >> These numbers point out that binary is faster by 6%, csv is slower by
> >> 5%, while text stays around what looks like noise range. That's not
> >> negligible. Are these numbers reproducible? If they are, that could
> >> be a problem for anybody doing bulk-loading of large data sets. I am
> >> not sure to understand where the improvement for binary comes from by
> >> reading the patch, but perhaps perf would tell more for each format?
> >> The loss with csv could be blamed on the extra manipulations of the
> >> function pointers, likely.
> >
> >
> > I don't think that's at all acceptable.
> >
> > We've spent quite a lot of blood sweat and tears over the years to make COPY
> > fast, and we should not sacrifice any of that lightly.
>
> These numbers aren't reproducible. Because these benchmarks
> executed on my normal machine not a machine only for
> benchmarking. The machine runs another processes such as
> editor and Web browser.
>
> For example, here are some results with master
> (94edfe250c6a200d2067b0debfe00b4122e9b11e):
>
> Format,N records,Elapsed time (ms)
> csv,10000000,1073.715
> csv,10000000,1022.830
> csv,10000000,1073.584
> csv,10000000,1090.651
> csv,10000000,1052.259
>
> Here are some results with master + the 0001 patch:
>
> Format,N records,Elapsed time (ms)
> csv,10000000,1025.356
> csv,10000000,1067.202
> csv,10000000,1014.563
> csv,10000000,1032.088
> csv,10000000,1058.110
>
>
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
>
> Could anyone try the benchmark with master and master+0001?
>
I've run a similar scenario:
create unlogged table test (a int);
insert into test select c from generate_series(1, 25000000) c;
copy test to '/tmp/result.csv' with (format csv); -- generates 230MB file
I've run it on HEAD and HEAD+0001 patch and here are the medians of 10
executions for each format:
HEAD:
binary 2930.353 ms
text 2754.852 ms
csv 2890.012 ms
HEAD w/ 0001 patch:
binary 2814.838 ms
text 2900.845 ms
csv 3015.210 ms
Hmm I can see a similar trend that Suto-san had; the binary format got
slightly faster whereas both text and csv format has small regression
(4%~5%). I think that the improvement for binary came from the fact
that we removed "if (cstate->opts.binary)" branches from the original
CopyOneRowTo(). I've experimented with a similar optimization for csv
and text format; have different callbacks for text and csv format and
remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
for that. Here are results:
HEAD w/ 0001 patch + remove branches:
binary 2824.502 ms
text 2715.264 ms
csv 2803.381 ms
The numbers look better now. I'm not sure these are within a noise
range but it might be worth considering having different callbacks for
text and csv formats.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] add_callback_for_csv_format.patch (3.9K, ../../CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com/2-add_callback_for_csv_format.patch)
download | inline diff:
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6547b7c654..f18b7d0823 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -223,11 +223,7 @@ CopyToTextStart(CopyToState cstate, TupleDesc tupDesc)
colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, colname, false,
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, colname);
+ CopyAttributeOutText(cstate, colname);
}
CopyToTextSendEndOfRow(cstate);
@@ -260,12 +256,7 @@ CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot)
char *string;
string = OutputFunctionCall(&out_functions[attnum - 1], value);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, string,
- cstate->opts.force_quote_flags[attnum - 1],
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, string);
+ CopyAttributeOutText(cstate, string);
}
}
@@ -277,6 +268,99 @@ CopyToTextEnd(CopyToState cstate)
{
}
+static void
+CopyToCSVStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ /*
+ * For non-binary copy, we need to convert null_print to file encoding,
+ * because it will be sent directly with CopySendString.
+ */
+ if (cstate->need_transcoding)
+ cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
+ cstate->opts.null_print_len,
+ cstate->file_encoding);
+
+ /* if a header has been requested send the line */
+ if (cstate->opts.header_line)
+ {
+ bool hdr_delim = false;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ char *colname;
+
+ if (hdr_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ hdr_delim = true;
+
+ colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
+
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ bool need_delim = false;
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (need_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ need_delim = true;
+
+ if (isnull)
+ {
+ CopySendString(cstate, cstate->opts.null_print_client);
+ }
+ else
+ {
+ char *string;
+
+ string = OutputFunctionCall(&out_functions[attnum - 1], value);
+ CopyAttributeOutCSV(cstate, string,
+ cstate->opts.force_quote_flags[attnum - 1],
+ list_length(cstate->attnumlist) == 1);
+ }
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+}
+
+static void
+CopyToCSVEnd(CopyToState cstate)
+{
+}
+
/*
* CopyToRoutine implementation for "binary".
*/
@@ -388,9 +472,9 @@ CopyToRoutine CopyToRoutineText = {
CopyToRoutine CopyToRoutineCSV = {
.CopyToProcessOption = CopyToTextProcessOption,
.CopyToGetFormat = CopyToTextGetFormat,
- .CopyToStart = CopyToTextStart,
- .CopyToOneRow = CopyToTextOneRow,
- .CopyToEnd = CopyToTextEnd,
+ .CopyToStart = CopyToCSVStart,
+ .CopyToOneRow = CopyToCSVOneRow,
+ .CopyToEnd = CopyToCSVEnd,
};
CopyToRoutine CopyToRoutineBinary = {
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-25 04:53 ` Michael Paquier <[email protected]>
2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-25 04:53 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote:
> Hmm I can see a similar trend that Suto-san had; the binary format got
> slightly faster whereas both text and csv format has small regression
> (4%~5%). I think that the improvement for binary came from the fact
> that we removed "if (cstate->opts.binary)" branches from the original
> CopyOneRowTo(). I've experimented with a similar optimization for csv
> and text format; have different callbacks for text and csv format and
> remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> for that. Here are results:
>
> HEAD w/ 0001 patch + remove branches:
> binary 2824.502 ms
> text 2715.264 ms
> csv 2803.381 ms
>
> The numbers look better now. I'm not sure these are within a noise
> range but it might be worth considering having different callbacks for
> text and csv formats.
Interesting.
Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and
0.9% speedup for binary, which may be around the noise range assuming
a ~1% range. While this does not imply a regression, that seems worth
the duplication IMO. The patch had better document the reason why the
split is done, as well.
CopyFromTextOneRow() has also specific branches for binary and
non-binary removed in 0005, so assuming that I/O is not a bottleneck,
the operation would be faster because we would not evaluate this "if"
condition for each row. Wouldn't we also see improvements for COPY
FROM with short row values, say when mounting PGDATA into a
tmpfs/ramfs?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-25 05:28 ` Masahiko Sawada <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Masahiko Sawada @ 2024-01-25 05:28 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 1:53 PM Michael Paquier <[email protected]> wrote:
>
> On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote:
> > Hmm I can see a similar trend that Suto-san had; the binary format got
> > slightly faster whereas both text and csv format has small regression
> > (4%~5%). I think that the improvement for binary came from the fact
> > that we removed "if (cstate->opts.binary)" branches from the original
> > CopyOneRowTo(). I've experimented with a similar optimization for csv
> > and text format; have different callbacks for text and csv format and
> > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> > for that. Here are results:
> >
> > HEAD w/ 0001 patch + remove branches:
> > binary 2824.502 ms
> > text 2715.264 ms
> > csv 2803.381 ms
> >
> > The numbers look better now. I'm not sure these are within a noise
> > range but it might be worth considering having different callbacks for
> > text and csv formats.
>
> Interesting.
>
> Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and
> 0.9% speedup for binary, which may be around the noise range assuming
> a ~1% range. While this does not imply a regression, that seems worth
> the duplication IMO.
Agreed. In addition to that, now that each format routine has its own
callbacks, there would be chances that we can do other optimizations
dedicated to the format type in the future if available.
> The patch had better document the reason why the
> split is done, as well.
+1
>
> CopyFromTextOneRow() has also specific branches for binary and
> non-binary removed in 0005, so assuming that I/O is not a bottleneck,
> the operation would be faster because we would not evaluate this "if"
> condition for each row. Wouldn't we also see improvements for COPY
> FROM with short row values, say when mounting PGDATA into a
> tmpfs/ramfs?
Probably. Seems worth evaluating.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-25 08:52 ` Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:52 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900,
Masahiko Sawada <[email protected]> wrote:
> I've experimented with a similar optimization for csv
> and text format; have different callbacks for text and csv format and
> remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> for that. Here are results:
>
> HEAD w/ 0001 patch + remove branches:
> binary 2824.502 ms
> text 2715.264 ms
> csv 2803.381 ms
>
> The numbers look better now. I'm not sure these are within a noise
> range but it might be worth considering having different callbacks for
> text and csv formats.
Wow! Interesting. I tried the approach before but I didn't
see any difference by the approach. But it may depend on my
environment.
I'll import the approach to the next patch set so that
others can try the approach easily.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 08:18 ` Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Junwang Zhao @ 2024-01-26 08:18 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 4:52 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
> > I've experimented with a similar optimization for csv
> > and text format; have different callbacks for text and csv format and
> > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> > for that. Here are results:
> >
> > HEAD w/ 0001 patch + remove branches:
> > binary 2824.502 ms
> > text 2715.264 ms
> > csv 2803.381 ms
> >
> > The numbers look better now. I'm not sure these are within a noise
> > range but it might be worth considering having different callbacks for
> > text and csv formats.
>
> Wow! Interesting. I tried the approach before but I didn't
> see any difference by the approach. But it may depend on my
> environment.
>
> I'll import the approach to the next patch set so that
> others can try the approach easily.
>
>
> Thanks,
> --
> kou
Hi Kou-san,
In the current implementation, there is no way that one can check
incompatibility
options in ProcessCopyOptions, we can postpone the check in CopyFromStart
or CopyToStart, but I think it is a little bit late. Do you think
adding an extra
check for incompatible options hook is acceptable (PFA)?
--
Regards
Junwang Zhao
Attachments:
[application/octet-stream] 0001-add-check-incomptiblity-options-hooks.patch (3.1K, ../../CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com/2-0001-add-check-incomptiblity-options-hooks.patch)
download | inline diff:
From f01ad61fe6c251a00870ace5d37669350f8e2043 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Fri, 26 Jan 2024 15:55:07 +0800
Subject: [PATCH] add check incomptiblity options hooks
Signed-off-by: Zhao Junwang <[email protected]>
---
src/backend/commands/copy.c | 5 +++++
src/include/commands/copyapi.h | 15 +++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 479f36868c..985d50870f 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -924,6 +924,11 @@ ProcessCopyOptions(ParseState *pstate,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("NULL specification and DEFAULT specification cannot be the same")));
}
+
+ if (is_from)
+ opts_out->from_routine->CopyFromCheckIncompatibleOptions(cstate, opts_out);
+ else
+ opts_out->to_routine->CopyToCheckIncompatibleOptions(cstate, opts_out);
}
/*
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 22accc83ab..34aa86761a 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -21,8 +21,10 @@
#include "nodes/parsenodes.h"
typedef struct CopyFromStateData *CopyFromState;
+typedef struct CopyFormatOptions;
typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
+typedef void (*CopyFromCheckIncompatibleOptions_function) (CopyFromState cstate, CopyFormatOptions *options);
typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate);
typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc);
typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
@@ -39,6 +41,12 @@ typedef struct CopyFromRoutine
*/
CopyFromProcessOption_function CopyFromProcessOption;
+ /*
+ * Called for processing incompatible options. This will report error
+ * messages when find incompatible options.
+ */
+ CopyFromCheckIncompatibleOptions_function CopyFromCheckIncompatibleOptions;
+
/*
* Called when COPY FROM is started. This will return a format as int16
* value. It's used for the CopyInResponse message.
@@ -67,6 +75,7 @@ extern CopyFromRoutine CopyFromRoutineBinary;
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef void (*CopyToCheckIncompatibleOptions_function) (CopyToState cstate, CopyFormatOptions *options);
typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
@@ -83,6 +92,12 @@ typedef struct CopyToRoutine
*/
CopyToProcessOption_function CopyToProcessOption;
+ /*
+ * Called for processing incompatible options. This will report error
+ * messages when find incompatible options.
+ */
+ CopyToCheckIncompatibleOptions_function CopyToCheckIncompatibleOptions;
+
/*
* Called when COPY TO is started. This will return a format as int16
* value. It's used for the CopyOutResponse message.
--
2.41.0
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-26 08:32 ` Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:32 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800,
Junwang Zhao <[email protected]> wrote:
> In the current implementation, there is no way that one can check
> incompatibility
> options in ProcessCopyOptions, we can postpone the check in CopyFromStart
> or CopyToStart, but I think it is a little bit late. Do you think
> adding an extra
> check for incompatible options hook is acceptable (PFA)?
Thanks for the suggestion! But I think that a custom handler
can do it in
CopyToProcessOption()/CopyFromProcessOption(). What do you
think about this? Or could you share a sample COPY TO/FROM
WITH() SQL you think?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 08:41 ` Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Junwang Zhao @ 2024-01-26 08:41 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 4:32 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > In the current implementation, there is no way that one can check
> > incompatibility
> > options in ProcessCopyOptions, we can postpone the check in CopyFromStart
> > or CopyToStart, but I think it is a little bit late. Do you think
> > adding an extra
> > check for incompatible options hook is acceptable (PFA)?
>
> Thanks for the suggestion! But I think that a custom handler
> can do it in
> CopyToProcessOption()/CopyFromProcessOption(). What do you
> think about this? Or could you share a sample COPY TO/FROM
> WITH() SQL you think?
CopyToProcessOption()/CopyFromProcessOption() can only handle
single option, and store the options in the opaque field, but it can not
check the relation of two options, for example, considering json format,
the `header` option can not be handled by these two functions.
I want to find a way when the user specifies the header option, customer
handler can error out.
>
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-26 08:55 ` Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:55 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
Junwang Zhao <[email protected]> wrote:
> CopyToProcessOption()/CopyFromProcessOption() can only handle
> single option, and store the options in the opaque field, but it can not
> check the relation of two options, for example, considering json format,
> the `header` option can not be handled by these two functions.
>
> I want to find a way when the user specifies the header option, customer
> handler can error out.
Ah, you want to use a built-in option (such as "header")
value from a custom handler, right? Hmm, it may be better
that we call CopyToProcessOption()/CopyFromProcessOption()
for all options including built-in options.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 09:02 ` Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Junwang Zhao @ 2024-01-26 09:02 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > single option, and store the options in the opaque field, but it can not
> > check the relation of two options, for example, considering json format,
> > the `header` option can not be handled by these two functions.
> >
> > I want to find a way when the user specifies the header option, customer
> > handler can error out.
>
> Ah, you want to use a built-in option (such as "header")
> value from a custom handler, right? Hmm, it may be better
> that we call CopyToProcessOption()/CopyFromProcessOption()
> for all options including built-in options.
>
Hmm, still I don't think it can handle all cases, since we don't know
the sequence of the options, we need all the options been parsed
before we check the compatibility of the options, or customer
handlers will need complicated logic to resolve that, which might
lead to ugly code :(
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-27 06:15 ` Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Junwang Zhao @ 2024-01-27 06:15 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi Kou-san,
On Fri, Jan 26, 2024 at 5:02 PM Junwang Zhao <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > Junwang Zhao <[email protected]> wrote:
> >
> > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > single option, and store the options in the opaque field, but it can not
> > > check the relation of two options, for example, considering json format,
> > > the `header` option can not be handled by these two functions.
> > >
> > > I want to find a way when the user specifies the header option, customer
> > > handler can error out.
> >
> > Ah, you want to use a built-in option (such as "header")
> > value from a custom handler, right? Hmm, it may be better
> > that we call CopyToProcessOption()/CopyFromProcessOption()
> > for all options including built-in options.
> >
> Hmm, still I don't think it can handle all cases, since we don't know
> the sequence of the options, we need all the options been parsed
> before we check the compatibility of the options, or customer
> handlers will need complicated logic to resolve that, which might
> lead to ugly code :(
>
I have been working on a *COPY TO JSON* extension since yesterday,
which is based on your V6 patch set, I'd like to give you more input
so you can make better decisions about the implementation(with only
pg-copy-arrow you might not get everything considered).
V8 is based on V6, so anybody involved in the performance issue
should still review the V7 patch set.
0001-0008 is your original V6 implementations
0009 is some changes made by me, I changed CopyToGetFormat to
CopyToSendCopyBegin because pg_copy_json need to send different bytes
in SendCopyBegin, get the format code along is not enough, I once had
a thought that may be we should merge SendCopyBegin/SendCopyEnd into
CopyToStart/CopyToEnd but I don't do that in this patch. I have also
exported more APIs for extension usage.
00010 is the pg_copy_json extension, I think this should be a good
case which can utilize the *extendable copy format* feature, maybe we
should delete copy_test_format if we have this extension as an
example?
> >
> > Thanks,
> > --
> > kou
>
>
>
> --
> Regards
> Junwang Zhao
--
Regards
Junwang Zhao
Attachments:
[application/octet-stream] v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch (3.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/2-v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch)
download | inline diff:
From 4b177469f3fb8f14f0cd6bff3c7878dcafd9b760 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 15:12:43 +0900
Subject: [PATCH v8 04/10] Add support for implementing custom COPY TO format
as extension
* Add CopyToStateData::opaque that can be used to keep data for custom
COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
method for CopyToState and it's used for flushing. End-of-row related
codes were moved to CopyToTextSendEndOfRow().
---
src/backend/commands/copyto.c | 15 +++++++--------
src/include/commands/copyapi.h | 5 +++++
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cfc74ee7b1..b5d8678394 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate);
static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
static void CopySendString(CopyToState cstate, const char *str);
static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
static void CopySendInt32(CopyToState cstate, int32 val);
static void CopySendInt16(CopyToState cstate, int16 val);
@@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
default:
break;
}
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
static void
@@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
}
}
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
static void
@@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate)
/* Generate trailer for a binary copy */
CopySendInt16(cstate, -1);
/* Need to flush out the trailer */
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
CopyToRoutine CopyToRoutineText = {
@@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate)
* CopySendData sends output data to the destination (file or frontend)
* CopySendString does the same for null-terminated strings
* CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- * (data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ * (data is not actually flushed except by CopyToStateFlush)
*
* NB: no data conversion is applied by these functions
*----------
@@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c)
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
}
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
{
StringInfo fe_msgbuf = cstate->fe_msgbuf;
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index a869d78d72..ffad433a21 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -174,6 +174,11 @@ typedef struct CopyToStateData
FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* For custom format implementation */
+ void *opaque; /* private space */
} CopyToStateData;
+extern void CopyToStateFlush(CopyToState cstate);
+
#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0001-Extract-COPY-TO-format-implementations.patch (23.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/3-v8-0001-Extract-COPY-TO-format-implementations.patch)
download | inline diff:
From 6e68ba6380dc825a242e7f0d0a53442bba3a4a61 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Mon, 4 Dec 2023 12:32:54 +0900
Subject: [PATCH v8 01/10] Extract COPY TO format implementations
This is a part of making COPY format extendable. See also these past
discussions:
* New Copy Formats - avro/orc/parquet:
https://www.postgresql.org/message-id/flat/20180210151304.fonjztsynewldfba%40gmail.com
* Make COPY extendable in order to support Parquet and other formats:
https://www.postgresql.org/message-id/flat/CAJ7c6TM6Bz1c3F04Cy6%2BSzuWfKmr0kU8c_3Stnvh_8BR0D6k8Q%40mail.gmail.com
This doesn't change the current behavior. This just introduces
CopyToRoutine, which just has function pointers of format
implementation like TupleTableSlotOps, and use it for existing "text",
"csv" and "binary" format implementations.
Note that CopyToRoutine can't be used from extensions yet because
CopySend*() aren't exported yet. Extensions can't send formatted data
to a destination without CopySend*(). They will be exported by
subsequent patches.
Here is a benchmark result with/without this change because there was
a discussion that we should care about performance regression:
https://www.postgresql.org/message-id/3741749.1655952719%40sss.pgh.pa.us
> I think that step 1 ought to be to convert the existing formats into
> plug-ins, and demonstrate that there's no significant loss of
> performance.
You can see that there is no significant loss of performance:
Data: Random 32 bit integers:
CREATE TABLE data (int32 integer);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${n_records});
The number of records: 100K, 1M and 10M
100K without this change:
format,elapsed time (ms)
text,11.002
csv,11.696
binary,11.352
100K with this change:
format,elapsed time (ms)
text,100000,11.562
csv,100000,11.889
binary,100000,10.825
1M without this change:
format,elapsed time (ms)
text,108.359
csv,114.233
binary,111.251
1M with this change:
format,elapsed time (ms)
text,111.269
csv,116.277
binary,104.765
10M without this change:
format,elapsed time (ms)
text,1090.763
csv,1136.103
binary,1137.141
10M with this change:
format,elapsed time (ms)
text,1082.654
csv,1196.991
binary,1069.697
---
contrib/file_fdw/file_fdw.c | 2 +-
src/backend/commands/copy.c | 43 +++-
src/backend/commands/copyfrom.c | 2 +-
src/backend/commands/copyto.c | 428 ++++++++++++++++++++------------
src/include/commands/copy.h | 7 +-
src/include/commands/copyapi.h | 59 +++++
6 files changed, 376 insertions(+), 165 deletions(-)
create mode 100644 src/include/commands/copyapi.h
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 249d82d3a0..9e4e819858 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -329,7 +329,7 @@ file_fdw_validator(PG_FUNCTION_ARGS)
/*
* Now apply the core COPY code's validation logic for more checks.
*/
- ProcessCopyOptions(NULL, NULL, true, other_options);
+ ProcessCopyOptions(NULL, NULL, true, NULL, other_options);
/*
* Either filename or program option is required for file_fdw foreign
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index cc0786c6f4..5f3697a5f9 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -442,6 +442,9 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from)
* a list of options. In that usage, 'opts_out' can be passed as NULL and
* the collected data is just leaked until CurrentMemoryContext is reset.
*
+ * 'cstate' is CopyToState* for !is_from, CopyFromState* for is_from. 'cstate'
+ * may be NULL. For example, file_fdw uses NULL.
+ *
* Note that additional checking, such as whether column names listed in FORCE
* QUOTE actually exist, has to be applied later. This just checks for
* self-consistency of the options list.
@@ -450,6 +453,7 @@ void
ProcessCopyOptions(ParseState *pstate,
CopyFormatOptions *opts_out,
bool is_from,
+ void *cstate,
List *options)
{
bool format_specified = false;
@@ -464,7 +468,13 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->file_encoding = -1;
- /* Extract options from the statement node tree */
+ /* Text is the default format. */
+ opts_out->to_routine = &CopyToRoutineText;
+
+ /*
+ * Extract only the "format" option to detect target routine as the first
+ * step
+ */
foreach(option, options)
{
DefElem *defel = lfirst_node(DefElem, option);
@@ -479,15 +489,29 @@ ProcessCopyOptions(ParseState *pstate,
if (strcmp(fmt, "text") == 0)
/* default format */ ;
else if (strcmp(fmt, "csv") == 0)
+ {
opts_out->csv_mode = true;
+ opts_out->to_routine = &CopyToRoutineCSV;
+ }
else if (strcmp(fmt, "binary") == 0)
+ {
opts_out->binary = true;
+ opts_out->to_routine = &CopyToRoutineBinary;
+ }
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY format \"%s\" not recognized", fmt),
parser_errposition(pstate, defel->location)));
}
+ }
+ /* Extract options except "format" from the statement node tree */
+ foreach(option, options)
+ {
+ DefElem *defel = lfirst_node(DefElem, option);
+
+ if (strcmp(defel->defname, "format") == 0)
+ continue;
else if (strcmp(defel->defname, "freeze") == 0)
{
if (freeze_specified)
@@ -616,11 +640,18 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->on_error = defGetCopyOnErrorChoice(defel, pstate, is_from);
}
else
- ereport(ERROR,
- (errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("option \"%s\" not recognized",
- defel->defname),
- parser_errposition(pstate, defel->location)));
+ {
+ bool processed = false;
+
+ if (!is_from)
+ processed = opts_out->to_routine->CopyToProcessOption(cstate, defel);
+ if (!processed)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("option \"%s\" not recognized",
+ defel->defname),
+ parser_errposition(pstate, defel->location)));
+ }
}
/*
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 1fe70b9133..fb3d4d9296 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1416,7 +1416,7 @@ BeginCopyFrom(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options);
+ ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , cstate, options);
/* Process the target relation */
cstate->rel = rel;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index d3dc3fc854..6547b7c654 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -131,6 +131,275 @@ static void CopySendEndOfRow(CopyToState cstate);
static void CopySendInt32(CopyToState cstate, int32 val);
static void CopySendInt16(CopyToState cstate, int16 val);
+/*
+ * CopyToRoutine implementations.
+ */
+
+/*
+ * CopyToRoutine implementation for "text" and "csv". CopyToText*()
+ * refer cstate->opts.csv_mode and change their behavior. We can split this
+ * implementation and stop referring cstate->opts.csv_mode later.
+ */
+
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
+static bool
+CopyToTextProcessOption(CopyToState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyToTextGetFormat(CopyToState cstate)
+{
+ return 0;
+}
+
+static void
+CopyToTextSendEndOfRow(CopyToState cstate)
+{
+ switch (cstate->copy_dest)
+ {
+ case COPY_FILE:
+ /* Default line termination depends on platform */
+#ifndef WIN32
+ CopySendChar(cstate, '\n');
+#else
+ CopySendString(cstate, "\r\n");
+#endif
+ break;
+ case COPY_FRONTEND:
+ /* The FE/BE protocol uses \n as newline for all platforms */
+ CopySendChar(cstate, '\n');
+ break;
+ default:
+ break;
+ }
+ CopySendEndOfRow(cstate);
+}
+
+static void
+CopyToTextStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ /*
+ * For non-binary copy, we need to convert null_print to file encoding,
+ * because it will be sent directly with CopySendString.
+ */
+ if (cstate->need_transcoding)
+ cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
+ cstate->opts.null_print_len,
+ cstate->file_encoding);
+
+ /* if a header has been requested send the line */
+ if (cstate->opts.header_line)
+ {
+ bool hdr_delim = false;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ char *colname;
+
+ if (hdr_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ hdr_delim = true;
+
+ colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
+
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, colname);
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ bool need_delim = false;
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (need_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ need_delim = true;
+
+ if (isnull)
+ {
+ CopySendString(cstate, cstate->opts.null_print_client);
+ }
+ else
+ {
+ char *string;
+
+ string = OutputFunctionCall(&out_functions[attnum - 1], value);
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, string,
+ cstate->opts.force_quote_flags[attnum - 1],
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, string);
+ }
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+}
+
+static void
+CopyToTextEnd(CopyToState cstate)
+{
+}
+
+/*
+ * CopyToRoutine implementation for "binary".
+ */
+
+/* All "binary" options are parsed in ProcessCopyOptions(). We may move the
+ * code to here later. */
+static bool
+CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyToBinaryGetFormat(CopyToState cstate)
+{
+ return 1;
+}
+
+static void
+CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ {
+ /* Generate header for a binary copy */
+ int32 tmp;
+
+ /* Signature */
+ CopySendData(cstate, BinarySignature, 11);
+ /* Flags field */
+ tmp = 0;
+ CopySendInt32(cstate, tmp);
+ /* No header extension */
+ tmp = 0;
+ CopySendInt32(cstate, tmp);
+ }
+}
+
+static void
+CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ /* Binary per-tuple header */
+ CopySendInt16(cstate, list_length(cstate->attnumlist));
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (isnull)
+ {
+ CopySendInt32(cstate, -1);
+ }
+ else
+ {
+ bytea *outputbytes;
+
+ outputbytes = SendFunctionCall(&out_functions[attnum - 1], value);
+ CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
+ CopySendData(cstate, VARDATA(outputbytes),
+ VARSIZE(outputbytes) - VARHDRSZ);
+ }
+ }
+
+ CopySendEndOfRow(cstate);
+}
+
+static void
+CopyToBinaryEnd(CopyToState cstate)
+{
+ /* Generate trailer for a binary copy */
+ CopySendInt16(cstate, -1);
+ /* Need to flush out the trailer */
+ CopySendEndOfRow(cstate);
+}
+
+CopyToRoutine CopyToRoutineText = {
+ .CopyToProcessOption = CopyToTextProcessOption,
+ .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToStart = CopyToTextStart,
+ .CopyToOneRow = CopyToTextOneRow,
+ .CopyToEnd = CopyToTextEnd,
+};
+
+/*
+ * We can use the same CopyToRoutine for both of "text" and "csv" because
+ * CopyToText*() refer cstate->opts.csv_mode and change their behavior. We can
+ * split the implementations and stop referring cstate->opts.csv_mode later.
+ */
+CopyToRoutine CopyToRoutineCSV = {
+ .CopyToProcessOption = CopyToTextProcessOption,
+ .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToStart = CopyToTextStart,
+ .CopyToOneRow = CopyToTextOneRow,
+ .CopyToEnd = CopyToTextEnd,
+};
+
+CopyToRoutine CopyToRoutineBinary = {
+ .CopyToProcessOption = CopyToBinaryProcessOption,
+ .CopyToGetFormat = CopyToBinaryGetFormat,
+ .CopyToStart = CopyToBinaryStart,
+ .CopyToOneRow = CopyToBinaryOneRow,
+ .CopyToEnd = CopyToBinaryEnd,
+};
/*
* Send copy start/stop messages for frontend copies. These have changed
@@ -141,7 +410,7 @@ SendCopyBegin(CopyToState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate);
int i;
pq_beginmessage(&buf, PqMsg_CopyOutResponse);
@@ -198,16 +467,6 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
case COPY_FILE:
- if (!cstate->opts.binary)
- {
- /* Default line termination depends on platform */
-#ifndef WIN32
- CopySendChar(cstate, '\n');
-#else
- CopySendString(cstate, "\r\n");
-#endif
- }
-
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
ferror(cstate->copy_file))
@@ -242,10 +501,6 @@ CopySendEndOfRow(CopyToState cstate)
}
break;
case COPY_FRONTEND:
- /* The FE/BE protocol uses \n as newline for all platforms */
- if (!cstate->opts.binary)
- CopySendChar(cstate, '\n');
-
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
@@ -431,7 +686,7 @@ BeginCopyTo(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options);
+ ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , cstate, options);
/* Process the source/target relation or query */
if (rel)
@@ -748,8 +1003,6 @@ DoCopyTo(CopyToState cstate)
bool pipe = (cstate->filename == NULL && cstate->data_dest_cb == NULL);
bool fe_copy = (pipe && whereToSendOutput == DestRemote);
TupleDesc tupDesc;
- int num_phys_attrs;
- ListCell *cur;
uint64 processed;
if (fe_copy)
@@ -759,32 +1012,11 @@ DoCopyTo(CopyToState cstate)
tupDesc = RelationGetDescr(cstate->rel);
else
tupDesc = cstate->queryDesc->tupDesc;
- num_phys_attrs = tupDesc->natts;
cstate->opts.null_print_client = cstate->opts.null_print; /* default */
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
cstate->fe_msgbuf = makeStringInfo();
- /* Get info about the columns we need to process. */
- cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- Oid out_func_oid;
- bool isvarlena;
- Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
-
- if (cstate->opts.binary)
- getTypeBinaryOutputInfo(attr->atttypid,
- &out_func_oid,
- &isvarlena);
- else
- getTypeOutputInfo(attr->atttypid,
- &out_func_oid,
- &isvarlena);
- fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
- }
-
/*
* Create a temporary memory context that we can reset once per row to
* recover palloc'd memory. This avoids any problems with leaks inside
@@ -795,57 +1027,7 @@ DoCopyTo(CopyToState cstate)
"COPY TO",
ALLOCSET_DEFAULT_SIZES);
- if (cstate->opts.binary)
- {
- /* Generate header for a binary copy */
- int32 tmp;
-
- /* Signature */
- CopySendData(cstate, BinarySignature, 11);
- /* Flags field */
- tmp = 0;
- CopySendInt32(cstate, tmp);
- /* No header extension */
- tmp = 0;
- CopySendInt32(cstate, tmp);
- }
- else
- {
- /*
- * For non-binary copy, we need to convert null_print to file
- * encoding, because it will be sent directly with CopySendString.
- */
- if (cstate->need_transcoding)
- cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
- cstate->opts.null_print_len,
- cstate->file_encoding);
-
- /* if a header has been requested send the line */
- if (cstate->opts.header_line)
- {
- bool hdr_delim = false;
-
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- char *colname;
-
- if (hdr_delim)
- CopySendChar(cstate, cstate->opts.delim[0]);
- hdr_delim = true;
-
- colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
-
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, colname, false,
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, colname);
- }
-
- CopySendEndOfRow(cstate);
- }
- }
+ cstate->opts.to_routine->CopyToStart(cstate, tupDesc);
if (cstate->rel)
{
@@ -884,13 +1066,7 @@ DoCopyTo(CopyToState cstate)
processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
}
- if (cstate->opts.binary)
- {
- /* Generate trailer for a binary copy */
- CopySendInt16(cstate, -1);
- /* Need to flush out the trailer */
- CopySendEndOfRow(cstate);
- }
+ cstate->opts.to_routine->CopyToEnd(cstate);
MemoryContextDelete(cstate->rowcontext);
@@ -906,71 +1082,15 @@ DoCopyTo(CopyToState cstate)
static void
CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
{
- bool need_delim = false;
- FmgrInfo *out_functions = cstate->out_functions;
MemoryContext oldcontext;
- ListCell *cur;
- char *string;
MemoryContextReset(cstate->rowcontext);
oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
- if (cstate->opts.binary)
- {
- /* Binary per-tuple header */
- CopySendInt16(cstate, list_length(cstate->attnumlist));
- }
-
/* Make sure the tuple is fully deconstructed */
slot_getallattrs(slot);
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- Datum value = slot->tts_values[attnum - 1];
- bool isnull = slot->tts_isnull[attnum - 1];
-
- if (!cstate->opts.binary)
- {
- if (need_delim)
- CopySendChar(cstate, cstate->opts.delim[0]);
- need_delim = true;
- }
-
- if (isnull)
- {
- if (!cstate->opts.binary)
- CopySendString(cstate, cstate->opts.null_print_client);
- else
- CopySendInt32(cstate, -1);
- }
- else
- {
- if (!cstate->opts.binary)
- {
- string = OutputFunctionCall(&out_functions[attnum - 1],
- value);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, string,
- cstate->opts.force_quote_flags[attnum - 1],
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, string);
- }
- else
- {
- bytea *outputbytes;
-
- outputbytes = SendFunctionCall(&out_functions[attnum - 1],
- value);
- CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
- CopySendData(cstate, VARDATA(outputbytes),
- VARSIZE(outputbytes) - VARHDRSZ);
- }
- }
- }
-
- CopySendEndOfRow(cstate);
+ cstate->opts.to_routine->CopyToOneRow(cstate, slot);
MemoryContextSwitchTo(oldcontext);
}
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index b3da3cb0be..34bea880ca 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -14,6 +14,7 @@
#ifndef COPY_H
#define COPY_H
+#include "commands/copyapi.h"
#include "nodes/execnodes.h"
#include "nodes/parsenodes.h"
#include "parser/parse_node.h"
@@ -74,11 +75,11 @@ typedef struct CopyFormatOptions
bool convert_selectively; /* do selective binary conversion? */
CopyOnErrorChoice on_error; /* what to do when error happened */
List *convert_select; /* list of column names (can be NIL) */
+ CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
-/* These are private in commands/copy[from|to].c */
+/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
-typedef struct CopyToStateData *CopyToState;
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
typedef void (*copy_data_dest_cb) (void *data, int len);
@@ -87,7 +88,7 @@ extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
uint64 *processed);
-extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options);
+extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options);
extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *whereClause,
const char *filename,
bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
new file mode 100644
index 0000000000..eb68f2fb7b
--- /dev/null
+++ b/src/include/commands/copyapi.h
@@ -0,0 +1,59 @@
+/*-------------------------------------------------------------------------
+ *
+ * copyapi.h
+ * API for COPY TO/FROM handlers
+ *
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/commands/copyapi.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef COPYAPI_H
+#define COPYAPI_H
+
+#include "executor/tuptable.h"
+#include "nodes/parsenodes.h"
+
+/* This is private in commands/copyto.c */
+typedef struct CopyToStateData *CopyToState;
+
+typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
+typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
+typedef void (*CopyToEnd_function) (CopyToState cstate);
+
+/* Routines for a COPY TO format implementation. */
+typedef struct CopyToRoutine
+{
+ /*
+ * Called for processing one COPY TO option. This will return false when
+ * the given option is invalid.
+ */
+ CopyToProcessOption_function CopyToProcessOption;
+
+ /*
+ * Called when COPY TO is started. This will return a format as int16
+ * value. It's used for the CopyOutResponse message.
+ */
+ CopyToGetFormat_function CopyToGetFormat;
+
+ /* Called when COPY TO is started. This will send a header. */
+ CopyToStart_function CopyToStart;
+
+ /* Copy one row for COPY TO. */
+ CopyToOneRow_function CopyToOneRow;
+
+ /* Called when COPY TO is ended. This will send a trailer. */
+ CopyToEnd_function CopyToEnd;
+} CopyToRoutine;
+
+/* Built-in CopyToRoutine for "text", "csv" and "binary". */
+extern CopyToRoutine CopyToRoutineText;
+extern CopyToRoutine CopyToRoutineCSV;
+extern CopyToRoutine CopyToRoutineBinary;
+
+#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch (17.7K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/4-v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch)
download | inline diff:
From a597f8a2beec12971d77419f08b5722f531774f3 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 13:58:38 +0900
Subject: [PATCH v8 02/10] Add support for adding custom COPY TO format
This uses the handler approach like tablesample. The approach creates
an internal function that returns an internal struct. In this case,
a COPY TO handler returns a CopyToRoutine.
We will add support for custom COPY FROM format later. We'll use the
same handler for COPY TO and COPY FROM. PostgreSQL calls a COPY
TO/FROM handler with "is_from" argument. It's true for COPY FROM and
false for COPY TO:
copy_handler(true) returns CopyToRoutine
copy_handler(false) returns CopyFromRoutine (not exist yet)
We discussed that we introduce a wrapper struct for it:
typedef struct CopyRoutine
{
NodeTag type;
/* either CopyToRoutine or CopyFromRoutine */
Node *routine;
}
copy_handler(true) returns CopyRoutine with CopyToRoutine
copy_handler(false) returns CopyRoutine with CopyFromRoutine
See also: https://www.postgresql.org/message-id/flat/CAD21AoCunywHird3GaPzWe6s9JG1wzxj3Cr6vGN36DDheGjOjA%40mail.gmail.com
But I noticed that we don't need the wrapper struct. We can just
CopyToRoutine or CopyFromRoutine. Because we can distinct the returned
struct by checking its NodeTag. So I don't use the wrapper struct
approach.
---
src/backend/commands/copy.c | 84 ++++++++++++++-----
src/backend/nodes/Makefile | 1 +
src/backend/nodes/gen_node_support.pl | 2 +
src/backend/utils/adt/pseudotypes.c | 1 +
src/include/catalog/pg_proc.dat | 6 ++
src/include/catalog/pg_type.dat | 6 ++
src/include/commands/copyapi.h | 2 +
src/include/nodes/meson.build | 1 +
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
src/test/modules/test_copy_format/.gitignore | 4 +
src/test/modules/test_copy_format/Makefile | 23 +++++
.../expected/test_copy_format.out | 17 ++++
src/test/modules/test_copy_format/meson.build | 33 ++++++++
.../test_copy_format/sql/test_copy_format.sql | 8 ++
.../test_copy_format--1.0.sql | 8 ++
.../test_copy_format/test_copy_format.c | 77 +++++++++++++++++
.../test_copy_format/test_copy_format.control | 4 +
18 files changed, 260 insertions(+), 19 deletions(-)
mode change 100644 => 100755 src/backend/nodes/gen_node_support.pl
create mode 100644 src/test/modules/test_copy_format/.gitignore
create mode 100644 src/test/modules/test_copy_format/Makefile
create mode 100644 src/test/modules/test_copy_format/expected/test_copy_format.out
create mode 100644 src/test/modules/test_copy_format/meson.build
create mode 100644 src/test/modules/test_copy_format/sql/test_copy_format.sql
create mode 100644 src/test/modules/test_copy_format/test_copy_format--1.0.sql
create mode 100644 src/test/modules/test_copy_format/test_copy_format.c
create mode 100644 src/test/modules/test_copy_format/test_copy_format.control
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 5f3697a5f9..6f0db0ae7c 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -32,6 +32,7 @@
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
#include "parser/parse_expr.h"
+#include "parser/parse_func.h"
#include "parser/parse_relation.h"
#include "rewrite/rewriteHandler.h"
#include "utils/acl.h"
@@ -430,6 +431,69 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from)
return COPY_ON_ERROR_STOP; /* keep compiler quiet */
}
+/*
+ * Process the "format" option.
+ *
+ * This function checks whether the option value is a built-in format such as
+ * "text" and "csv" or not. If the option value isn't a built-in format, this
+ * function finds a COPY format handler that returns a CopyToRoutine. If no
+ * COPY format handler is found, this function reports an error.
+ */
+static void
+ProcessCopyOptionCustomFormat(ParseState *pstate,
+ CopyFormatOptions *opts_out,
+ bool is_from,
+ DefElem *defel)
+{
+ char *format;
+ Oid funcargtypes[1];
+ Oid handlerOid = InvalidOid;
+ Datum datum;
+ void *routine;
+
+ format = defGetString(defel);
+
+ /* built-in formats */
+ if (strcmp(format, "text") == 0)
+ /* default format */ return;
+ else if (strcmp(format, "csv") == 0)
+ {
+ opts_out->csv_mode = true;
+ opts_out->to_routine = &CopyToRoutineCSV;
+ return;
+ }
+ else if (strcmp(format, "binary") == 0)
+ {
+ opts_out->binary = true;
+ opts_out->to_routine = &CopyToRoutineBinary;
+ return;
+ }
+
+ /* custom format */
+ if (!is_from)
+ {
+ funcargtypes[0] = INTERNALOID;
+ handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
+ funcargtypes, true);
+ }
+ if (!OidIsValid(handlerOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", format),
+ parser_errposition(pstate, defel->location)));
+
+ datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from));
+ routine = DatumGetPointer(datum);
+ if (routine == NULL || !IsA(routine, CopyToRoutine))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct",
+ format, handlerOid),
+ parser_errposition(pstate, defel->location)));
+
+ opts_out->to_routine = routine;
+}
+
/*
* Process the statement option list for COPY.
*
@@ -481,28 +545,10 @@ ProcessCopyOptions(ParseState *pstate,
if (strcmp(defel->defname, "format") == 0)
{
- char *fmt = defGetString(defel);
-
if (format_specified)
errorConflictingDefElem(defel, pstate);
format_specified = true;
- if (strcmp(fmt, "text") == 0)
- /* default format */ ;
- else if (strcmp(fmt, "csv") == 0)
- {
- opts_out->csv_mode = true;
- opts_out->to_routine = &CopyToRoutineCSV;
- }
- else if (strcmp(fmt, "binary") == 0)
- {
- opts_out->binary = true;
- opts_out->to_routine = &CopyToRoutineBinary;
- }
- else
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format \"%s\" not recognized", fmt),
- parser_errposition(pstate, defel->location)));
+ ProcessCopyOptionCustomFormat(pstate, opts_out, is_from, defel);
}
}
/* Extract options except "format" from the statement node tree */
diff --git a/src/backend/nodes/Makefile b/src/backend/nodes/Makefile
index 66bbad8e6e..173ee11811 100644
--- a/src/backend/nodes/Makefile
+++ b/src/backend/nodes/Makefile
@@ -49,6 +49,7 @@ node_headers = \
access/sdir.h \
access/tableam.h \
access/tsmapi.h \
+ commands/copyapi.h \
commands/event_trigger.h \
commands/trigger.h \
executor/tuptable.h \
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
old mode 100644
new mode 100755
index 2f0a59bc87..bd397f45ac
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -61,6 +61,7 @@ my @all_input_files = qw(
access/sdir.h
access/tableam.h
access/tsmapi.h
+ commands/copyapi.h
commands/event_trigger.h
commands/trigger.h
executor/tuptable.h
@@ -85,6 +86,7 @@ my @nodetag_only_files = qw(
access/sdir.h
access/tableam.h
access/tsmapi.h
+ commands/copyapi.h
commands/event_trigger.h
commands/trigger.h
executor/tuptable.h
diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c
index a3a991f634..d308780c43 100644
--- a/src/backend/utils/adt/pseudotypes.c
+++ b/src/backend/utils/adt/pseudotypes.c
@@ -373,6 +373,7 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(fdw_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(table_am_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(index_am_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(tsm_handler);
+PSEUDOTYPE_DUMMY_IO_FUNCS(copy_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(internal);
PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement);
PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 29af4ce65d..d4e426687c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7617,6 +7617,12 @@
{ oid => '3312', descr => 'I/O',
proname => 'tsm_handler_out', prorettype => 'cstring',
proargtypes => 'tsm_handler', prosrc => 'tsm_handler_out' },
+{ oid => '8753', descr => 'I/O',
+ proname => 'copy_handler_in', proisstrict => 'f', prorettype => 'copy_handler',
+ proargtypes => 'cstring', prosrc => 'copy_handler_in' },
+{ oid => '8754', descr => 'I/O',
+ proname => 'copy_handler_out', prorettype => 'cstring',
+ proargtypes => 'copy_handler', prosrc => 'copy_handler_out' },
{ oid => '267', descr => 'I/O',
proname => 'table_am_handler_in', proisstrict => 'f',
prorettype => 'table_am_handler', proargtypes => 'cstring',
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index d29194da31..2040d5da83 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -632,6 +632,12 @@
typcategory => 'P', typinput => 'tsm_handler_in',
typoutput => 'tsm_handler_out', typreceive => '-', typsend => '-',
typalign => 'i' },
+{ oid => '8752',
+ descr => 'pseudo-type for the result of a copy to/from method functoin',
+ typname => 'copy_handler', typlen => '4', typbyval => 't', typtype => 'p',
+ typcategory => 'P', typinput => 'copy_handler_in',
+ typoutput => 'copy_handler_out', typreceive => '-', typsend => '-',
+ typalign => 'i' },
{ oid => '269',
typname => 'table_am_handler',
descr => 'pseudo-type for the result of a table AM handler function',
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index eb68f2fb7b..9c25e1c415 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -29,6 +29,8 @@ typedef void (*CopyToEnd_function) (CopyToState cstate);
/* Routines for a COPY TO format implementation. */
typedef struct CopyToRoutine
{
+ NodeTag type;
+
/*
* Called for processing one COPY TO option. This will return false when
* the given option is invalid.
diff --git a/src/include/nodes/meson.build b/src/include/nodes/meson.build
index b665e55b65..103df1a787 100644
--- a/src/include/nodes/meson.build
+++ b/src/include/nodes/meson.build
@@ -11,6 +11,7 @@ node_support_input_i = [
'access/sdir.h',
'access/tableam.h',
'access/tsmapi.h',
+ 'commands/copyapi.h',
'commands/event_trigger.h',
'commands/trigger.h',
'executor/tuptable.h',
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index e32c8925f6..9d57b868d5 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -15,6 +15,7 @@ SUBDIRS = \
spgist_name_ops \
test_bloomfilter \
test_copy_callbacks \
+ test_copy_format \
test_custom_rmgrs \
test_ddl_deparse \
test_dsa \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 397e0906e6..d76f2a6003 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -13,6 +13,7 @@ subdir('spgist_name_ops')
subdir('ssl_passphrase_callback')
subdir('test_bloomfilter')
subdir('test_copy_callbacks')
+subdir('test_copy_format')
subdir('test_custom_rmgrs')
subdir('test_ddl_deparse')
subdir('test_dsa')
diff --git a/src/test/modules/test_copy_format/.gitignore b/src/test/modules/test_copy_format/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_copy_format/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_copy_format/Makefile b/src/test/modules/test_copy_format/Makefile
new file mode 100644
index 0000000000..8497f91624
--- /dev/null
+++ b/src/test/modules/test_copy_format/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/test_copy_format/Makefile
+
+MODULE_big = test_copy_format
+OBJS = \
+ $(WIN32RES) \
+ test_copy_format.o
+PGFILEDESC = "test_copy_format - test custom COPY FORMAT"
+
+EXTENSION = test_copy_format
+DATA = test_copy_format--1.0.sql
+
+REGRESS = test_copy_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_copy_format
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
new file mode 100644
index 0000000000..3a24ae7b97
--- /dev/null
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -0,0 +1,17 @@
+CREATE EXTENSION test_copy_format;
+CREATE TABLE public.test (a INT, b INT, c INT);
+INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test TO stdout WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+NOTICE: test_copy_format: is_from=false
+NOTICE: CopyToProcessOption: "option_before"="before"
+NOTICE: CopyToProcessOption: "option_after"="after"
+NOTICE: CopyToGetFormat
+NOTICE: CopyToStart: natts=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToEnd
diff --git a/src/test/modules/test_copy_format/meson.build b/src/test/modules/test_copy_format/meson.build
new file mode 100644
index 0000000000..4cefe7b709
--- /dev/null
+++ b/src/test/modules/test_copy_format/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+test_copy_format_sources = files(
+ 'test_copy_format.c',
+)
+
+if host_system == 'windows'
+ test_copy_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_copy_format',
+ '--FILEDESC', 'test_copy_format - test custom COPY FORMAT',])
+endif
+
+test_copy_format = shared_module('test_copy_format',
+ test_copy_format_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_copy_format
+
+test_install_data += files(
+ 'test_copy_format.control',
+ 'test_copy_format--1.0.sql',
+)
+
+tests += {
+ 'name': 'test_copy_format',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_copy_format',
+ ],
+ },
+}
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
new file mode 100644
index 0000000000..0eb7ed2e11
--- /dev/null
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -0,0 +1,8 @@
+CREATE EXTENSION test_copy_format;
+CREATE TABLE public.test (a INT, b INT, c INT);
+INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test TO stdout WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
diff --git a/src/test/modules/test_copy_format/test_copy_format--1.0.sql b/src/test/modules/test_copy_format/test_copy_format--1.0.sql
new file mode 100644
index 0000000000..d24ea03ce9
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format--1.0.sql
@@ -0,0 +1,8 @@
+/* src/test/modules/test_copy_format/test_copy_format--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_copy_format" to load this file. \quit
+
+CREATE FUNCTION test_copy_format(internal)
+ RETURNS copy_handler
+ AS 'MODULE_PATHNAME' LANGUAGE C;
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
new file mode 100644
index 0000000000..a2219afcde
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -0,0 +1,77 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_copy_format.c
+ * Code for testing custom COPY format.
+ *
+ * Portions Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_copy_format/test_copy_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/defrem.h"
+
+PG_MODULE_MAGIC;
+
+static bool
+CopyToProcessOption(CopyToState cstate, DefElem *defel)
+{
+ ereport(NOTICE,
+ (errmsg("CopyToProcessOption: \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel))));
+ return true;
+}
+
+static int16
+CopyToGetFormat(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToGetFormat")));
+ return 0;
+}
+
+static void
+CopyToStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ ereport(NOTICE, (errmsg("CopyToStart: natts=%d", tupDesc->natts)));
+}
+
+static void
+CopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", slot->tts_nvalid)));
+}
+
+static void
+CopyToEnd(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToEnd")));
+}
+
+static const CopyToRoutine CopyToRoutineTestCopyFormat = {
+ .type = T_CopyToRoutine,
+ .CopyToProcessOption = CopyToProcessOption,
+ .CopyToGetFormat = CopyToGetFormat,
+ .CopyToStart = CopyToStart,
+ .CopyToOneRow = CopyToOneRow,
+ .CopyToEnd = CopyToEnd,
+};
+
+PG_FUNCTION_INFO_V1(test_copy_format);
+Datum
+test_copy_format(PG_FUNCTION_ARGS)
+{
+ bool is_from = PG_GETARG_BOOL(0);
+
+ ereport(NOTICE,
+ (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false")));
+
+ if (is_from)
+ elog(ERROR, "COPY FROM isn't supported yet");
+
+ PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
+}
diff --git a/src/test/modules/test_copy_format/test_copy_format.control b/src/test/modules/test_copy_format/test_copy_format.control
new file mode 100644
index 0000000000..f05a636235
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format.control
@@ -0,0 +1,4 @@
+comment = 'Test code for custom COPY format'
+default_version = '1.0'
+module_pathname = '$libdir/test_copy_format'
+relocatable = true
--
2.41.0
[application/octet-stream] v8-0003-Export-CopyToStateData.patch (13.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/5-v8-0003-Export-CopyToStateData.patch)
download | inline diff:
From c3a59753b1157dc8e47e719263f58677acc33178 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 14:54:10 +0900
Subject: [PATCH v8 03/10] Export CopyToStateData
It's for custom COPY TO format handlers implemented as extension.
This just moves codes. This doesn't change codes except CopyDest enum
values. CopyDest enum values such as COPY_FILE are conflicted
CopySource enum values defined in copyfrom_internal.h. So COPY_DEST_
prefix instead of COPY_ prefix is used. For example, COPY_FILE is
renamed to COPY_DEST_FILE.
Note that this change isn't enough to implement a custom COPY TO
format handler as extension. We'll do the followings in a subsequent
commit:
1. Add an opaque space for custom COPY TO format handler
2. Export CopySendEndOfRow() to flush buffer
---
src/backend/commands/copyto.c | 74 +++-----------------
src/include/commands/copy.h | 59 ----------------
src/include/commands/copyapi.h | 120 ++++++++++++++++++++++++++++++++-
3 files changed, 127 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6547b7c654..cfc74ee7b1 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -43,64 +43,6 @@
#include "utils/rel.h"
#include "utils/snapmgr.h"
-/*
- * Represents the different dest cases we need to worry about at
- * the bottom level
- */
-typedef enum CopyDest
-{
- COPY_FILE, /* to file (or a piped program) */
- COPY_FRONTEND, /* to frontend */
- COPY_CALLBACK, /* to callback function */
-} CopyDest;
-
-/*
- * This struct contains all the state variables used throughout a COPY TO
- * operation.
- *
- * Multi-byte encodings: all supported client-side encodings encode multi-byte
- * characters by having the first byte's high bit set. Subsequent bytes of the
- * character can have the high bit not set. When scanning data in such an
- * encoding to look for a match to a single-byte (ie ASCII) character, we must
- * use the full pg_encoding_mblen() machinery to skip over multibyte
- * characters, else we might find a false match to a trailing byte. In
- * supported server encodings, there is no possibility of a false match, and
- * it's faster to make useless comparisons to trailing bytes than it is to
- * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
- * when we have to do it the hard way.
- */
-typedef struct CopyToStateData
-{
- /* low-level state data */
- CopyDest copy_dest; /* type of copy source/destination */
- FILE *copy_file; /* used if copy_dest == COPY_FILE */
- StringInfo fe_msgbuf; /* used for all dests during COPY TO */
-
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- FmgrInfo *out_functions; /* lookup info for output functions */
- MemoryContext rowcontext; /* per-row evaluation context */
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
-
/* DestReceiver for COPY (query) TO */
typedef struct
{
@@ -160,7 +102,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
{
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
/* Default line termination depends on platform */
#ifndef WIN32
CopySendChar(cstate, '\n');
@@ -168,7 +110,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
CopySendString(cstate, "\r\n");
#endif
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* The FE/BE protocol uses \n as newline for all platforms */
CopySendChar(cstate, '\n');
break;
@@ -419,7 +361,7 @@ SendCopyBegin(CopyToState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->copy_dest = COPY_DEST_FRONTEND;
}
static void
@@ -466,7 +408,7 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
ferror(cstate->copy_file))
@@ -500,11 +442,11 @@ CopySendEndOfRow(CopyToState cstate)
errmsg("could not write to COPY file: %m")));
}
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
- case COPY_CALLBACK:
+ case COPY_DEST_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
break;
}
@@ -877,12 +819,12 @@ BeginCopyTo(ParseState *pstate,
/* See Multibyte encoding comment above */
cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->copy_dest = COPY_DEST_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->copy_dest = COPY_DEST_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 34bea880ca..b3f4682f95 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,69 +20,10 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-/*
- * Represents whether a header line should be present, and whether it must
- * match the actual names (which implies "true").
- */
-typedef enum CopyHeaderChoice
-{
- COPY_HEADER_FALSE = 0,
- COPY_HEADER_TRUE,
- COPY_HEADER_MATCH,
-} CopyHeaderChoice;
-
-/*
- * Represents where to save input processing errors. More values to be added
- * in the future.
- */
-typedef enum CopyOnErrorChoice
-{
- COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */
- COPY_ON_ERROR_IGNORE, /* ignore errors */
-} CopyOnErrorChoice;
-
-/*
- * A struct to hold COPY options, in a parsed form. All of these are related
- * to formatting, except for 'freeze', which doesn't really belong here, but
- * it's expedient to parse it along with all the other options.
- */
-typedef struct CopyFormatOptions
-{
- /* parameters from the COPY command */
- int file_encoding; /* file or remote side's character encoding,
- * -1 if not specified */
- bool binary; /* binary format? */
- bool freeze; /* freeze rows on loading? */
- bool csv_mode; /* Comma Separated Value format? */
- CopyHeaderChoice header_line; /* header line? */
- char *null_print; /* NULL marker string (server encoding!) */
- int null_print_len; /* length of same */
- char *null_print_client; /* same converted to file encoding */
- char *default_print; /* DEFAULT marker string */
- int default_print_len; /* length of same */
- char *delim; /* column delimiter (must be 1 byte) */
- char *quote; /* CSV quote char (must be 1 byte) */
- char *escape; /* CSV escape char (must be 1 byte) */
- List *force_quote; /* list of column names */
- bool force_quote_all; /* FORCE_QUOTE *? */
- bool *force_quote_flags; /* per-column CSV FQ flags */
- List *force_notnull; /* list of column names */
- bool force_notnull_all; /* FORCE_NOT_NULL *? */
- bool *force_notnull_flags; /* per-column CSV FNN flags */
- List *force_null; /* list of column names */
- bool force_null_all; /* FORCE_NULL *? */
- bool *force_null_flags; /* per-column CSV FN flags */
- bool convert_selectively; /* do selective binary conversion? */
- CopyOnErrorChoice on_error; /* what to do when error happened */
- List *convert_select; /* list of column names (can be NIL) */
- CopyToRoutine *to_routine; /* callback routines for COPY TO */
-} CopyFormatOptions;
-
/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
-typedef void (*copy_data_dest_cb) (void *data, int len);
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 9c25e1c415..a869d78d72 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,10 +14,10 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "executor/execdesc.h"
#include "executor/tuptable.h"
#include "nodes/parsenodes.h"
-/* This is private in commands/copyto.c */
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
@@ -58,4 +58,122 @@ extern CopyToRoutine CopyToRoutineText;
extern CopyToRoutine CopyToRoutineCSV;
extern CopyToRoutine CopyToRoutineBinary;
+/*
+ * Represents whether a header line should be present, and whether it must
+ * match the actual names (which implies "true").
+ */
+typedef enum CopyHeaderChoice
+{
+ COPY_HEADER_FALSE = 0,
+ COPY_HEADER_TRUE,
+ COPY_HEADER_MATCH,
+} CopyHeaderChoice;
+
+/*
+ * Represents where to save input processing errors. More values to be added
+ * in the future.
+ */
+typedef enum CopyOnErrorChoice
+{
+ COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */
+ COPY_ON_ERROR_IGNORE, /* ignore errors */
+} CopyOnErrorChoice;
+
+/*
+ * A struct to hold COPY options, in a parsed form. All of these are related
+ * to formatting, except for 'freeze', which doesn't really belong here, but
+ * it's expedient to parse it along with all the other options.
+ */
+typedef struct CopyFormatOptions
+{
+ /* parameters from the COPY command */
+ int file_encoding; /* file or remote side's character encoding,
+ * -1 if not specified */
+ bool binary; /* binary format? */
+ bool freeze; /* freeze rows on loading? */
+ bool csv_mode; /* Comma Separated Value format? */
+ CopyHeaderChoice header_line; /* header line? */
+ char *null_print; /* NULL marker string (server encoding!) */
+ int null_print_len; /* length of same */
+ char *null_print_client; /* same converted to file encoding */
+ char *default_print; /* DEFAULT marker string */
+ int default_print_len; /* length of same */
+ char *delim; /* column delimiter (must be 1 byte) */
+ char *quote; /* CSV quote char (must be 1 byte) */
+ char *escape; /* CSV escape char (must be 1 byte) */
+ List *force_quote; /* list of column names */
+ bool force_quote_all; /* FORCE_QUOTE *? */
+ bool *force_quote_flags; /* per-column CSV FQ flags */
+ List *force_notnull; /* list of column names */
+ bool force_notnull_all; /* FORCE_NOT_NULL *? */
+ bool *force_notnull_flags; /* per-column CSV FNN flags */
+ List *force_null; /* list of column names */
+ bool force_null_all; /* FORCE_NULL *? */
+ bool *force_null_flags; /* per-column CSV FN flags */
+ bool convert_selectively; /* do selective binary conversion? */
+ CopyOnErrorChoice on_error; /* what to do when error happened */
+ List *convert_select; /* list of column names (can be NIL) */
+ CopyToRoutine *to_routine; /* callback routines for COPY TO */
+} CopyFormatOptions;
+
+/*
+ * Represents the different dest cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopyDest
+{
+ COPY_DEST_FILE, /* to file (or a piped program) */
+ COPY_DEST_FRONTEND, /* to frontend */
+ COPY_DEST_CALLBACK, /* to callback function */
+} CopyDest;
+
+typedef void (*copy_data_dest_cb) (void *data, int len);
+
+/*
+ * This struct contains all the state variables used throughout a COPY TO
+ * operation.
+ *
+ * Multi-byte encodings: all supported client-side encodings encode multi-byte
+ * characters by having the first byte's high bit set. Subsequent bytes of the
+ * character can have the high bit not set. When scanning data in such an
+ * encoding to look for a match to a single-byte (ie ASCII) character, we must
+ * use the full pg_encoding_mblen() machinery to skip over multibyte
+ * characters, else we might find a false match to a trailing byte. In
+ * supported server encodings, there is no possibility of a false match, and
+ * it's faster to make useless comparisons to trailing bytes than it is to
+ * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
+ * when we have to do it the hard way.
+ */
+typedef struct CopyToStateData
+{
+ /* low-level state data */
+ CopyDest copy_dest; /* type of copy source/destination */
+ FILE *copy_file; /* used if copy_dest == COPY_FILE */
+ StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+ MemoryContext rowcontext; /* per-row evaluation context */
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyToStateData;
+
#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0005-Extract-COPY-FROM-format-implementations.patch (24.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/6-v8-0005-Extract-COPY-FROM-format-implementations.patch)
download | inline diff:
From 781955f19ad27cdd66748be539bf45cf1b925856 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 17:21:23 +0900
Subject: [PATCH v8 05/10] Extract COPY FROM format implementations
This doesn't change the current behavior. This just introduces
CopyFromRoutine, which just has function pointers of format
implementation like TupleTableSlotOps, and use it for existing "text",
"csv" and "binary" format implementations.
Note that CopyFromRoutine can't be used from extensions yet because
CopyRead*() aren't exported yet. Extensions can't read data from a
source without CopyRead*(). They will be exported by subsequent
patches.
---
src/backend/commands/copy.c | 3 +
src/backend/commands/copyfrom.c | 216 +++++++++++----
src/backend/commands/copyfromparse.c | 326 ++++++++++++-----------
src/include/commands/copy.h | 3 -
src/include/commands/copyapi.h | 44 +++
src/include/commands/copyfrom_internal.h | 4 +
6 files changed, 391 insertions(+), 205 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 6f0db0ae7c..ec6dfff8ab 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -459,12 +459,14 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
else if (strcmp(format, "csv") == 0)
{
opts_out->csv_mode = true;
+ opts_out->from_routine = &CopyFromRoutineCSV;
opts_out->to_routine = &CopyToRoutineCSV;
return;
}
else if (strcmp(format, "binary") == 0)
{
opts_out->binary = true;
+ opts_out->from_routine = &CopyFromRoutineBinary;
opts_out->to_routine = &CopyToRoutineBinary;
return;
}
@@ -533,6 +535,7 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->file_encoding = -1;
/* Text is the default format. */
+ opts_out->from_routine = &CopyFromRoutineText;
opts_out->to_routine = &CopyToRoutineText;
/*
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index fb3d4d9296..d556ebb5d6 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -108,6 +108,170 @@ static char *limit_printout_length(const char *str);
static void ClosePipeFromProgram(CopyFromState cstate);
+
+/*
+ * CopyFromRoutine implementations.
+ */
+
+/*
+ * CopyFromRoutine implementation for "text" and "csv". CopyFromText*()
+ * refer cstate->opts.csv_mode and change their behavior. We can split this
+ * implementation and stop referring cstate->opts.csv_mode later.
+ */
+
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
+static bool
+CopyFromTextProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyFromTextGetFormat(CopyFromState cstate)
+{
+ return 0;
+}
+
+static void
+CopyFromTextStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ AttrNumber num_phys_attrs = tupDesc->natts;
+ AttrNumber attr_count;
+
+ /*
+ * If encoding conversion is needed, we need another buffer to hold the
+ * converted input data. Otherwise, we can just point input_buf to the
+ * same buffer as raw_buf.
+ */
+ if (cstate->need_transcoding)
+ {
+ cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
+ cstate->input_buf_index = cstate->input_buf_len = 0;
+ }
+ else
+ cstate->input_buf = cstate->raw_buf;
+ cstate->input_reached_eof = false;
+
+ initStringInfo(&cstate->line_buf);
+
+ /*
+ * Pick up the required catalog information for each attribute in the
+ * relation, including the input function, the element type (to pass to
+ * the input function).
+ */
+ cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
+ for (int attnum = 1; attnum <= num_phys_attrs; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
+ Oid in_func_oid;
+
+ /* We don't need info for dropped attributes */
+ if (att->attisdropped)
+ continue;
+
+ /* Fetch the input function and typioparam info */
+ getTypeInputInfo(att->atttypid,
+ &in_func_oid, &cstate->typioparams[attnum - 1]);
+ fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]);
+ }
+
+ /* create workspace for CopyReadAttributes results */
+ attr_count = list_length(cstate->attnumlist);
+ cstate->max_fields = attr_count;
+ cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
+}
+
+static void
+CopyFromTextEnd(CopyFromState cstate)
+{
+}
+
+/*
+ * CopyFromRoutine implementation for "binary".
+ */
+
+/* All "binary" options are parsed in ProcessCopyOptions(). We may move the
+ * code to here later. */
+static bool
+CopyFromBinaryProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyFromBinaryGetFormat(CopyFromState cstate)
+{
+ return 1;
+}
+
+static void
+CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ AttrNumber num_phys_attrs = tupDesc->natts;
+
+ /*
+ * Pick up the required catalog information for each attribute in the
+ * relation, including the input function, the element type (to pass to
+ * the input function).
+ */
+ cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
+ for (int attnum = 1; attnum <= num_phys_attrs; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
+ Oid in_func_oid;
+
+ /* We don't need info for dropped attributes */
+ if (att->attisdropped)
+ continue;
+
+ /* Fetch the input function and typioparam info */
+ getTypeBinaryInputInfo(att->atttypid,
+ &in_func_oid, &cstate->typioparams[attnum - 1]);
+ fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]);
+ }
+
+ /* Read and verify binary header */
+ ReceiveCopyBinaryHeader(cstate);
+}
+
+static void
+CopyFromBinaryEnd(CopyFromState cstate)
+{
+}
+
+CopyFromRoutine CopyFromRoutineText = {
+ .CopyFromProcessOption = CopyFromTextProcessOption,
+ .CopyFromGetFormat = CopyFromTextGetFormat,
+ .CopyFromStart = CopyFromTextStart,
+ .CopyFromOneRow = CopyFromTextOneRow,
+ .CopyFromEnd = CopyFromTextEnd,
+};
+
+/*
+ * We can use the same CopyFromRoutine for both of "text" and "csv" because
+ * CopyFromText*() refer cstate->opts.csv_mode and change their behavior. We can
+ * split the implementations and stop referring cstate->opts.csv_mode later.
+ */
+CopyFromRoutine CopyFromRoutineCSV = {
+ .CopyFromProcessOption = CopyFromTextProcessOption,
+ .CopyFromGetFormat = CopyFromTextGetFormat,
+ .CopyFromStart = CopyFromTextStart,
+ .CopyFromOneRow = CopyFromTextOneRow,
+ .CopyFromEnd = CopyFromTextEnd,
+};
+
+CopyFromRoutine CopyFromRoutineBinary = {
+ .CopyFromProcessOption = CopyFromBinaryProcessOption,
+ .CopyFromGetFormat = CopyFromBinaryGetFormat,
+ .CopyFromStart = CopyFromBinaryStart,
+ .CopyFromOneRow = CopyFromBinaryOneRow,
+ .CopyFromEnd = CopyFromBinaryEnd,
+};
+
+
/*
* error context callback for COPY FROM
*
@@ -1384,9 +1548,6 @@ BeginCopyFrom(ParseState *pstate,
TupleDesc tupDesc;
AttrNumber num_phys_attrs,
num_defaults;
- FmgrInfo *in_functions;
- Oid *typioparams;
- Oid in_func_oid;
int *defmap;
ExprState **defexprs;
MemoryContext oldcontext;
@@ -1571,25 +1732,6 @@ BeginCopyFrom(ParseState *pstate,
cstate->raw_buf_index = cstate->raw_buf_len = 0;
cstate->raw_reached_eof = false;
- if (!cstate->opts.binary)
- {
- /*
- * If encoding conversion is needed, we need another buffer to hold
- * the converted input data. Otherwise, we can just point input_buf
- * to the same buffer as raw_buf.
- */
- if (cstate->need_transcoding)
- {
- cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
- cstate->input_buf_index = cstate->input_buf_len = 0;
- }
- else
- cstate->input_buf = cstate->raw_buf;
- cstate->input_reached_eof = false;
-
- initStringInfo(&cstate->line_buf);
- }
-
initStringInfo(&cstate->attribute_buf);
/* Assign range table and rteperminfos, we'll need them in CopyFrom. */
@@ -1608,8 +1750,6 @@ BeginCopyFrom(ParseState *pstate,
* the input function), and info about defaults and constraints. (Which
* input function we use depends on text/binary format choice.)
*/
- in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
- typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
defmap = (int *) palloc(num_phys_attrs * sizeof(int));
defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
@@ -1621,15 +1761,6 @@ BeginCopyFrom(ParseState *pstate,
if (att->attisdropped)
continue;
- /* Fetch the input function and typioparam info */
- if (cstate->opts.binary)
- getTypeBinaryInputInfo(att->atttypid,
- &in_func_oid, &typioparams[attnum - 1]);
- else
- getTypeInputInfo(att->atttypid,
- &in_func_oid, &typioparams[attnum - 1]);
- fmgr_info(in_func_oid, &in_functions[attnum - 1]);
-
/* Get default info if available */
defexprs[attnum - 1] = NULL;
@@ -1689,8 +1820,6 @@ BeginCopyFrom(ParseState *pstate,
cstate->bytes_processed = 0;
/* We keep those variables in cstate. */
- cstate->in_functions = in_functions;
- cstate->typioparams = typioparams;
cstate->defmap = defmap;
cstate->defexprs = defexprs;
cstate->volatile_defexprs = volatile_defexprs;
@@ -1763,20 +1892,7 @@ BeginCopyFrom(ParseState *pstate,
pgstat_progress_update_multi_param(3, progress_cols, progress_vals);
- if (cstate->opts.binary)
- {
- /* Read and verify binary header */
- ReceiveCopyBinaryHeader(cstate);
- }
-
- /* create workspace for CopyReadAttributes results */
- if (!cstate->opts.binary)
- {
- AttrNumber attr_count = list_length(cstate->attnumlist);
-
- cstate->max_fields = attr_count;
- cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
- }
+ cstate->opts.from_routine->CopyFromStart(cstate, tupDesc);
MemoryContextSwitchTo(oldcontext);
@@ -1789,6 +1905,8 @@ BeginCopyFrom(ParseState *pstate,
void
EndCopyFrom(CopyFromState cstate)
{
+ cstate->opts.from_routine->CopyFromEnd(cstate);
+
/* No COPY FROM related resources except memory. */
if (cstate->is_program)
{
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 7cacd0b752..49632f75e4 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -172,7 +172,7 @@ ReceiveCopyBegin(CopyFromState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = cstate->opts.from_routine->CopyFromGetFormat(cstate);
int i;
pq_beginmessage(&buf, PqMsg_CopyInResponse);
@@ -840,199 +840,219 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
return true;
}
-/*
- * Read next tuple from file for COPY FROM. Return false if no more tuples.
- *
- * 'econtext' is used to evaluate default expression for each column that is
- * either not read from the file or is using the DEFAULT option of COPY FROM.
- * It can be NULL when no default values are used, i.e. when all columns are
- * read from the file, and DEFAULT option is unset.
- *
- * 'values' and 'nulls' arrays must be the same length as columns of the
- * relation passed to BeginCopyFrom. This function fills the arrays.
- */
bool
-NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls)
+CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
{
TupleDesc tupDesc;
- AttrNumber num_phys_attrs,
- attr_count,
- num_defaults = cstate->num_defaults;
+ AttrNumber attr_count;
FmgrInfo *in_functions = cstate->in_functions;
Oid *typioparams = cstate->typioparams;
- int i;
- int *defmap = cstate->defmap;
ExprState **defexprs = cstate->defexprs;
+ char **field_strings;
+ ListCell *cur;
+ int fldct;
+ int fieldno;
+ char *string;
tupDesc = RelationGetDescr(cstate->rel);
- num_phys_attrs = tupDesc->natts;
attr_count = list_length(cstate->attnumlist);
- /* Initialize all values for row to NULL */
- MemSet(values, 0, num_phys_attrs * sizeof(Datum));
- MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ /* read raw fields in the next line */
+ if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
+ return false;
- if (!cstate->opts.binary)
- {
- char **field_strings;
- ListCell *cur;
- int fldct;
- int fieldno;
- char *string;
+ /* check for overflowing fields */
+ if (attr_count > 0 && fldct > attr_count)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("extra data after last expected column")));
- /* read raw fields in the next line */
- if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
- return false;
+ fieldno = 0;
- /* check for overflowing fields */
- if (attr_count > 0 && fldct > attr_count)
+ /* Loop to read the user attributes on the line. */
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ int m = attnum - 1;
+ Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+ if (fieldno >= fldct)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("extra data after last expected column")));
+ errmsg("missing data for column \"%s\"",
+ NameStr(att->attname))));
+ string = field_strings[fieldno++];
- fieldno = 0;
-
- /* Loop to read the user attributes on the line. */
- foreach(cur, cstate->attnumlist)
+ if (cstate->convert_select_flags &&
+ !cstate->convert_select_flags[m])
{
- int attnum = lfirst_int(cur);
- int m = attnum - 1;
- Form_pg_attribute att = TupleDescAttr(tupDesc, m);
-
- if (fieldno >= fldct)
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("missing data for column \"%s\"",
- NameStr(att->attname))));
- string = field_strings[fieldno++];
-
- if (cstate->convert_select_flags &&
- !cstate->convert_select_flags[m])
- {
- /* ignore input field, leaving column as NULL */
- continue;
- }
+ /* ignore input field, leaving column as NULL */
+ continue;
+ }
- if (cstate->opts.csv_mode)
+ if (cstate->opts.csv_mode)
+ {
+ if (string == NULL &&
+ cstate->opts.force_notnull_flags[m])
{
- if (string == NULL &&
- cstate->opts.force_notnull_flags[m])
- {
- /*
- * FORCE_NOT_NULL option is set and column is NULL -
- * convert it to the NULL string.
- */
- string = cstate->opts.null_print;
- }
- else if (string != NULL && cstate->opts.force_null_flags[m]
- && strcmp(string, cstate->opts.null_print) == 0)
- {
- /*
- * FORCE_NULL option is set and column matches the NULL
- * string. It must have been quoted, or otherwise the
- * string would already have been set to NULL. Convert it
- * to NULL as specified.
- */
- string = NULL;
- }
+ /*
+ * FORCE_NOT_NULL option is set and column is NULL - convert
+ * it to the NULL string.
+ */
+ string = cstate->opts.null_print;
}
-
- cstate->cur_attname = NameStr(att->attname);
- cstate->cur_attval = string;
-
- if (string != NULL)
- nulls[m] = false;
-
- if (cstate->defaults[m])
+ else if (string != NULL && cstate->opts.force_null_flags[m]
+ && strcmp(string, cstate->opts.null_print) == 0)
{
/*
- * The caller must supply econtext and have switched into the
- * per-tuple memory context in it.
+ * FORCE_NULL option is set and column matches the NULL
+ * string. It must have been quoted, or otherwise the string
+ * would already have been set to NULL. Convert it to NULL as
+ * specified.
*/
- Assert(econtext != NULL);
- Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
-
- values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
+ string = NULL;
}
+ }
+
+ cstate->cur_attname = NameStr(att->attname);
+ cstate->cur_attval = string;
+
+ if (string != NULL)
+ nulls[m] = false;
+ if (cstate->defaults[m])
+ {
/*
- * If ON_ERROR is specified with IGNORE, skip rows with soft
- * errors
+ * The caller must supply econtext and have switched into the
+ * per-tuple memory context in it.
*/
- else if (!InputFunctionCallSafe(&in_functions[m],
- string,
- typioparams[m],
- att->atttypmod,
- (Node *) cstate->escontext,
- &values[m]))
- {
- cstate->num_errors++;
- return true;
- }
+ Assert(econtext != NULL);
+ Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
+ values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
}
- Assert(fieldno == attr_count);
+ /*
+ * If ON_ERROR is specified with IGNORE, skip rows with soft errors
+ */
+ else if (!InputFunctionCallSafe(&in_functions[m],
+ string,
+ typioparams[m],
+ att->atttypmod,
+ (Node *) cstate->escontext,
+ &values[m]))
+ {
+ cstate->num_errors++;
+ return true;
+ }
+
+ cstate->cur_attname = NULL;
+ cstate->cur_attval = NULL;
}
- else
- {
- /* binary */
- int16 fld_count;
- ListCell *cur;
- cstate->cur_lineno++;
+ Assert(fieldno == attr_count);
- if (!CopyGetInt16(cstate, &fld_count))
- {
- /* EOF detected (end of file, or protocol-level EOF) */
- return false;
- }
+ return true;
+}
- if (fld_count == -1)
- {
- /*
- * Received EOF marker. Wait for the protocol-level EOF, and
- * complain if it doesn't come immediately. In COPY FROM STDIN,
- * this ensures that we correctly handle CopyFail, if client
- * chooses to send that now. When copying from file, we could
- * ignore the rest of the file like in text mode, but we choose to
- * be consistent with the COPY FROM STDIN case.
- */
- char dummy;
+bool
+CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ TupleDesc tupDesc;
+ AttrNumber attr_count;
+ FmgrInfo *in_functions = cstate->in_functions;
+ Oid *typioparams = cstate->typioparams;
+ int16 fld_count;
+ ListCell *cur;
- if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("received copy data after EOF marker")));
- return false;
- }
+ tupDesc = RelationGetDescr(cstate->rel);
+ attr_count = list_length(cstate->attnumlist);
- if (fld_count != attr_count)
+ cstate->cur_lineno++;
+
+ if (!CopyGetInt16(cstate, &fld_count))
+ {
+ /* EOF detected (end of file, or protocol-level EOF) */
+ return false;
+ }
+
+ if (fld_count == -1)
+ {
+ /*
+ * Received EOF marker. Wait for the protocol-level EOF, and complain
+ * if it doesn't come immediately. In COPY FROM STDIN, this ensures
+ * that we correctly handle CopyFail, if client chooses to send that
+ * now. When copying from file, we could ignore the rest of the file
+ * like in text mode, but we choose to be consistent with the COPY
+ * FROM STDIN case.
+ */
+ char dummy;
+
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("row field count is %d, expected %d",
- (int) fld_count, attr_count)));
+ errmsg("received copy data after EOF marker")));
+ return false;
+ }
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- int m = attnum - 1;
- Form_pg_attribute att = TupleDescAttr(tupDesc, m);
-
- cstate->cur_attname = NameStr(att->attname);
- values[m] = CopyReadBinaryAttribute(cstate,
- &in_functions[m],
- typioparams[m],
- att->atttypmod,
- &nulls[m]);
- cstate->cur_attname = NULL;
- }
+ if (fld_count != attr_count)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("row field count is %d, expected %d",
+ (int) fld_count, attr_count)));
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ int m = attnum - 1;
+ Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+ cstate->cur_attname = NameStr(att->attname);
+ values[m] = CopyReadBinaryAttribute(cstate,
+ &in_functions[m],
+ typioparams[m],
+ att->atttypmod,
+ &nulls[m]);
+ cstate->cur_attname = NULL;
}
+ return true;
+}
+
+/*
+ * Read next tuple from file for COPY FROM. Return false if no more tuples.
+ *
+ * 'econtext' is used to evaluate default expression for each column that is
+ * either not read from the file or is using the DEFAULT option of COPY FROM.
+ * It can be NULL when no default values are used, i.e. when all columns are
+ * read from the file, and DEFAULT option is unset.
+ *
+ * 'values' and 'nulls' arrays must be the same length as columns of the
+ * relation passed to BeginCopyFrom. This function fills the arrays.
+ */
+bool
+NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
+ Datum *values, bool *nulls)
+{
+ TupleDesc tupDesc;
+ AttrNumber num_phys_attrs,
+ num_defaults = cstate->num_defaults;
+ int i;
+ int *defmap = cstate->defmap;
+ ExprState **defexprs = cstate->defexprs;
+
+ tupDesc = RelationGetDescr(cstate->rel);
+ num_phys_attrs = tupDesc->natts;
+
+ /* Initialize all values for row to NULL */
+ MemSet(values, 0, num_phys_attrs * sizeof(Datum));
+ MemSet(nulls, true, num_phys_attrs * sizeof(bool));
+ MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+
+ if (!cstate->opts.from_routine->CopyFromOneRow(cstate, econtext, values,
+ nulls))
+ return false;
+
/*
* Now compute and insert any defaults available for the columns not
* provided by the input data. Anything not processed here or above will
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index b3f4682f95..df29d42555 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,9 +20,6 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-/* This is private in commands/copyfrom.c */
-typedef struct CopyFromStateData *CopyFromState;
-
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index ffad433a21..323e4705d2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -18,6 +18,49 @@
#include "executor/tuptable.h"
#include "nodes/parsenodes.h"
+/* This is private in commands/copyfrom.c */
+typedef struct CopyFromStateData *CopyFromState;
+
+typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
+typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate);
+typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc);
+typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+typedef void (*CopyFromEnd_function) (CopyFromState cstate);
+
+/* Routines for a COPY FROM format implementation. */
+typedef struct CopyFromRoutine
+{
+ /*
+ * Called for processing one COPY FROM option. This will return false when
+ * the given option is invalid.
+ */
+ CopyFromProcessOption_function CopyFromProcessOption;
+
+ /*
+ * Called when COPY FROM is started. This will return a format as int16
+ * value. It's used for the CopyInResponse message.
+ */
+ CopyFromGetFormat_function CopyFromGetFormat;
+
+ /*
+ * Called when COPY FROM is started. This will initialize something and
+ * receive a header.
+ */
+ CopyFromStart_function CopyFromStart;
+
+ /* Copy one row. It returns false if no more tuples. */
+ CopyFromOneRow_function CopyFromOneRow;
+
+ /* Called when COPY FROM is ended. This will finalize something. */
+ CopyFromEnd_function CopyFromEnd;
+} CopyFromRoutine;
+
+/* Built-in CopyFromRoutine for "text", "csv" and "binary". */
+extern CopyFromRoutine CopyFromRoutineText;
+extern CopyFromRoutine CopyFromRoutineCSV;
+extern CopyFromRoutine CopyFromRoutineBinary;
+
+
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
@@ -113,6 +156,7 @@ typedef struct CopyFormatOptions
bool convert_selectively; /* do selective binary conversion? */
CopyOnErrorChoice on_error; /* what to do when error happened */
List *convert_select; /* list of column names (can be NIL) */
+ CopyFromRoutine *from_routine; /* callback routines for COPY FROM */
CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index cad52fcc78..921c1513f7 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -183,4 +183,8 @@ typedef struct CopyFromStateData
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
+extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+
+
#endif /* COPYFROM_INTERNAL_H */
--
2.41.0
[application/octet-stream] v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch (7.6K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/7-v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch)
download | inline diff:
From f0a8151feff44823881c3c4e1e7aca4f9bd690d5 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Sat, 27 Jan 2024 09:53:31 +0800
Subject: [PATCH v8 09/10] change CopyToGetFormat to CopyToSendCopyBegin and
export more api
Signed-off-by: Zhao Junwang <[email protected]>
---
src/backend/commands/copyto.c | 65 ++++++++++---------
src/include/commands/copyapi.h | 12 ++--
.../test_copy_format/test_copy_format.c | 7 +-
3 files changed, 46 insertions(+), 38 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b5d8678394..e2a4964015 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -66,11 +66,6 @@ static void CopyAttributeOutCSV(CopyToState cstate, const char *string,
/* Low-level communications functions */
static void SendCopyBegin(CopyToState cstate);
static void SendCopyEnd(CopyToState cstate);
-static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
-static void CopySendString(CopyToState cstate, const char *str);
-static void CopySendChar(CopyToState cstate, char c);
-static void CopySendInt32(CopyToState cstate, int32 val);
-static void CopySendInt16(CopyToState cstate, int16 val);
/*
* CopyToRoutine implementations.
@@ -90,10 +85,20 @@ CopyToTextProcessOption(CopyToState cstate, DefElem *defel)
return false;
}
-static int16
-CopyToTextGetFormat(CopyToState cstate)
+static void
+CopyToTextSendCopyBegin(CopyToState cstate)
{
- return 0;
+ StringInfoData buf;
+ int natts = list_length(cstate->attnumlist);
+ int16 format = 0;
+ int i;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ pq_sendint16(&buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(&buf, format); /* per-column formats */
+ pq_endmessage(&buf);
}
static void
@@ -230,10 +235,20 @@ CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel)
return false;
}
-static int16
-CopyToBinaryGetFormat(CopyToState cstate)
+static void
+CopyToBinarySendCopyBegin(CopyToState cstate)
{
- return 1;
+ StringInfoData buf;
+ int natts = list_length(cstate->attnumlist);
+ int16 format = 1;
+ int i;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ pq_sendint16(&buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(&buf, format); /* per-column formats */
+ pq_endmessage(&buf);
}
static void
@@ -315,7 +330,7 @@ CopyToBinaryEnd(CopyToState cstate)
CopyToRoutine CopyToRoutineText = {
.CopyToProcessOption = CopyToTextProcessOption,
- .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToSendCopyBegin = CopyToTextSendCopyBegin,
.CopyToStart = CopyToTextStart,
.CopyToOneRow = CopyToTextOneRow,
.CopyToEnd = CopyToTextEnd,
@@ -328,7 +343,7 @@ CopyToRoutine CopyToRoutineText = {
*/
CopyToRoutine CopyToRoutineCSV = {
.CopyToProcessOption = CopyToTextProcessOption,
- .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToSendCopyBegin = CopyToTextSendCopyBegin,
.CopyToStart = CopyToTextStart,
.CopyToOneRow = CopyToTextOneRow,
.CopyToEnd = CopyToTextEnd,
@@ -336,7 +351,7 @@ CopyToRoutine CopyToRoutineCSV = {
CopyToRoutine CopyToRoutineBinary = {
.CopyToProcessOption = CopyToBinaryProcessOption,
- .CopyToGetFormat = CopyToBinaryGetFormat,
+ .CopyToSendCopyBegin = CopyToBinarySendCopyBegin,
.CopyToStart = CopyToBinaryStart,
.CopyToOneRow = CopyToBinaryOneRow,
.CopyToEnd = CopyToBinaryEnd,
@@ -349,17 +364,7 @@ CopyToRoutine CopyToRoutineBinary = {
static void
SendCopyBegin(CopyToState cstate)
{
- StringInfoData buf;
- int natts = list_length(cstate->attnumlist);
- int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate);
- int i;
-
- pq_beginmessage(&buf, PqMsg_CopyOutResponse);
- pq_sendbyte(&buf, format); /* overall format */
- pq_sendint16(&buf, natts);
- for (i = 0; i < natts; i++)
- pq_sendint16(&buf, format); /* per-column formats */
- pq_endmessage(&buf);
+ cstate->opts.to_routine->CopyToSendCopyBegin(cstate);
cstate->copy_dest = COPY_DEST_FRONTEND;
}
@@ -382,19 +387,19 @@ SendCopyEnd(CopyToState cstate)
* NB: no data conversion is applied by these functions
*----------
*/
-static void
+void
CopySendData(CopyToState cstate, const void *databuf, int datasize)
{
appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
}
-static void
+void
CopySendString(CopyToState cstate, const char *str)
{
appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
}
-static void
+void
CopySendChar(CopyToState cstate, char c)
{
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
@@ -464,7 +469,7 @@ CopyToStateFlush(CopyToState cstate)
/*
* CopySendInt32 sends an int32 in network byte order
*/
-static inline void
+inline void
CopySendInt32(CopyToState cstate, int32 val)
{
uint32 buf;
@@ -476,7 +481,7 @@ CopySendInt32(CopyToState cstate, int32 val)
/*
* CopySendInt16 sends an int16 in network byte order
*/
-static inline void
+inline void
CopySendInt16(CopyToState cstate, int16 val)
{
uint16 buf;
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 22accc83ab..0a05b24c54 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -67,7 +67,7 @@ extern CopyFromRoutine CopyFromRoutineBinary;
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
-typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToSendCopyBegin_function) (CopyToState cstate);
typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
typedef void (*CopyToEnd_function) (CopyToState cstate);
@@ -84,10 +84,9 @@ typedef struct CopyToRoutine
CopyToProcessOption_function CopyToProcessOption;
/*
- * Called when COPY TO is started. This will return a format as int16
- * value. It's used for the CopyOutResponse message.
+ * Called when COPY TO is started.
*/
- CopyToGetFormat_function CopyToGetFormat;
+ CopyToSendCopyBegin_function CopyToSendCopyBegin;
/* Called when COPY TO is started. This will send a header. */
CopyToStart_function CopyToStart;
@@ -384,6 +383,11 @@ typedef struct CopyToStateData
void *opaque; /* private space */
} CopyToStateData;
+extern void CopySendData(CopyToState cstate, const void *databuf, int datasize);
+extern void CopySendString(CopyToState cstate, const char *str);
+extern void CopySendChar(CopyToState cstate, char c);
+extern void CopySendInt32(CopyToState cstate, int32 val);
+extern void CopySendInt16(CopyToState cstate, int16 val);
extern void CopyToStateFlush(CopyToState cstate);
#endif /* COPYAPI_H */
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 5e1b40e881..d833f22bbf 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -71,11 +71,10 @@ CopyToProcessOption(CopyToState cstate, DefElem *defel)
return true;
}
-static int16
-CopyToGetFormat(CopyToState cstate)
+static void
+CopyToSendCopyBegin(CopyToState cstate)
{
ereport(NOTICE, (errmsg("CopyToGetFormat")));
- return 0;
}
static void
@@ -99,7 +98,7 @@ CopyToEnd(CopyToState cstate)
static const CopyToRoutine CopyToRoutineTestCopyFormat = {
.type = T_CopyToRoutine,
.CopyToProcessOption = CopyToProcessOption,
- .CopyToGetFormat = CopyToGetFormat,
+ .CopyToSendCopyBegin = CopyToSendCopyBegin,
.CopyToStart = CopyToStart,
.CopyToOneRow = CopyToOneRow,
.CopyToEnd = CopyToEnd,
--
2.41.0
[application/octet-stream] v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch (7.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/8-v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch)
download | inline diff:
From f48e7b629a8d15fc70cd4cc4737dd2ad61910cc9 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 11:07:14 +0900
Subject: [PATCH v8 06/10] Add support for adding custom COPY FROM format
We use the same approach as we used for custom COPY TO format. Now,
custom COPY format handler can return COPY TO format routines or COPY
FROM format routines based on the "is_from" argument:
copy_handler(true) returns CopyToRoutine
copy_handler(false) returns CopyFromRoutine
---
src/backend/commands/copy.c | 53 +++++++++++++------
src/include/commands/copyapi.h | 2 +
.../expected/test_copy_format.out | 12 +++++
.../test_copy_format/sql/test_copy_format.sql | 6 +++
.../test_copy_format/test_copy_format.c | 50 +++++++++++++++--
5 files changed, 105 insertions(+), 18 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index ec6dfff8ab..479f36868c 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -472,12 +472,9 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
}
/* custom format */
- if (!is_from)
- {
- funcargtypes[0] = INTERNALOID;
- handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
- funcargtypes, true);
- }
+ funcargtypes[0] = INTERNALOID;
+ handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
+ funcargtypes, true);
if (!OidIsValid(handlerOid))
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -486,14 +483,36 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from));
routine = DatumGetPointer(datum);
- if (routine == NULL || !IsA(routine, CopyToRoutine))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct",
- format, handlerOid),
- parser_errposition(pstate, defel->location)));
-
- opts_out->to_routine = routine;
+ if (is_from)
+ {
+ if (routine == NULL || !IsA(routine, CopyFromRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+ ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%s(%u) did not return a "
+ "CopyFromRoutine struct",
+ format, handlerOid),
+ parser_errposition(
+ pstate, defel->location)));
+ opts_out->from_routine = routine;
+ }
+ else
+ {
+ if (routine == NULL || !IsA(routine, CopyToRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+ ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%s(%u) did not return a "
+ "CopyToRoutine struct",
+ format, handlerOid),
+ parser_errposition(
+ pstate, defel->location)));
+ opts_out->to_routine = routine;
+ }
}
/*
@@ -692,7 +711,11 @@ ProcessCopyOptions(ParseState *pstate,
{
bool processed = false;
- if (!is_from)
+ if (is_from)
+ processed =
+ opts_out->from_routine->CopyFromProcessOption(
+ cstate, defel);
+ else
processed = opts_out->to_routine->CopyToProcessOption(cstate, defel);
if (!processed)
ereport(ERROR,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 323e4705d2..ef1bb201c2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -30,6 +30,8 @@ typedef void (*CopyFromEnd_function) (CopyFromState cstate);
/* Routines for a COPY FROM format implementation. */
typedef struct CopyFromRoutine
{
+ NodeTag type;
+
/*
* Called for processing one COPY FROM option. This will return false when
* the given option is invalid.
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
index 3a24ae7b97..6af69f0eb7 100644
--- a/src/test/modules/test_copy_format/expected/test_copy_format.out
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -1,6 +1,18 @@
CREATE EXTENSION test_copy_format;
CREATE TABLE public.test (a INT, b INT, c INT);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test FROM stdin WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+NOTICE: test_copy_format: is_from=true
+NOTICE: CopyFromProcessOption: "option_before"="before"
+NOTICE: CopyFromProcessOption: "option_after"="after"
+NOTICE: CopyFromGetFormat
+NOTICE: CopyFromStart: natts=3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
COPY public.test TO stdout WITH (
option_before 'before',
format 'test_copy_format',
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
index 0eb7ed2e11..94d3c789a0 100644
--- a/src/test/modules/test_copy_format/sql/test_copy_format.sql
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -1,6 +1,12 @@
CREATE EXTENSION test_copy_format;
CREATE TABLE public.test (a INT, b INT, c INT);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test FROM stdin WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+\.
COPY public.test TO stdout WITH (
option_before 'before',
format 'test_copy_format',
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index a2219afcde..5e1b40e881 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -18,6 +18,50 @@
PG_MODULE_MAGIC;
+static bool
+CopyFromProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ ereport(NOTICE,
+ (errmsg("CopyFromProcessOption: \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel))));
+ return true;
+}
+
+static int16
+CopyFromGetFormat(CopyFromState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyFromGetFormat")));
+ return 0;
+}
+
+static void
+CopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ ereport(NOTICE, (errmsg("CopyFromStart: natts=%d", tupDesc->natts)));
+}
+
+static bool
+CopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ ereport(NOTICE, (errmsg("CopyFromOneRow")));
+ return false;
+}
+
+static void
+CopyFromEnd(CopyFromState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyFromEnd")));
+}
+
+static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
+ .type = T_CopyFromRoutine,
+ .CopyFromProcessOption = CopyFromProcessOption,
+ .CopyFromGetFormat = CopyFromGetFormat,
+ .CopyFromStart = CopyFromStart,
+ .CopyFromOneRow = CopyFromOneRow,
+ .CopyFromEnd = CopyFromEnd,
+};
+
static bool
CopyToProcessOption(CopyToState cstate, DefElem *defel)
{
@@ -71,7 +115,7 @@ test_copy_format(PG_FUNCTION_ARGS)
(errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false")));
if (is_from)
- elog(ERROR, "COPY FROM isn't supported yet");
-
- PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
+ PG_RETURN_POINTER(&CopyFromRoutineTestCopyFormat);
+ else
+ PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
}
--
2.41.0
[application/octet-stream] v8-0010-introduce-contrib-pg_copy_json.patch (16.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/9-v8-0010-introduce-contrib-pg_copy_json.patch)
download | inline diff:
From 7dc6c1c798178f31728d048d4d528181626b3695 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Sat, 27 Jan 2024 13:34:38 +0800
Subject: [PATCH v8 10/10] introduce contrib/pg_copy_json
Signed-off-by: Zhao Junwang <[email protected]>
---
contrib/Makefile | 1 +
contrib/meson.build | 1 +
contrib/pg_copy_json/.gitignore | 4 +
contrib/pg_copy_json/Makefile | 23 ++
.../pg_copy_json/expected/pg_copy_json.out | 80 +++++++
contrib/pg_copy_json/meson.build | 34 +++
contrib/pg_copy_json/pg_copy_json--1.0.sql | 9 +
contrib/pg_copy_json/pg_copy_json.c | 218 ++++++++++++++++++
contrib/pg_copy_json/pg_copy_json.control | 5 +
contrib/pg_copy_json/sql/pg_copy_json.sql | 59 +++++
src/backend/utils/adt/json.c | 5 +-
src/include/utils/json.h | 2 +
12 files changed, 438 insertions(+), 3 deletions(-)
create mode 100644 contrib/pg_copy_json/.gitignore
create mode 100644 contrib/pg_copy_json/Makefile
create mode 100644 contrib/pg_copy_json/expected/pg_copy_json.out
create mode 100644 contrib/pg_copy_json/meson.build
create mode 100644 contrib/pg_copy_json/pg_copy_json--1.0.sql
create mode 100644 contrib/pg_copy_json/pg_copy_json.c
create mode 100644 contrib/pg_copy_json/pg_copy_json.control
create mode 100644 contrib/pg_copy_json/sql/pg_copy_json.sql
diff --git a/contrib/Makefile b/contrib/Makefile
index da4e2316a3..82cc496aa2 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -32,6 +32,7 @@ SUBDIRS = \
pageinspect \
passwordcheck \
pg_buffercache \
+ pg_copy_json \
pg_freespacemap \
pg_prewarm \
pg_stat_statements \
diff --git a/contrib/meson.build b/contrib/meson.build
index c12dc906ca..38933d15d1 100644
--- a/contrib/meson.build
+++ b/contrib/meson.build
@@ -45,6 +45,7 @@ subdir('oid2name')
subdir('pageinspect')
subdir('passwordcheck')
subdir('pg_buffercache')
+subdir('pg_copy_json')
subdir('pgcrypto')
subdir('pg_freespacemap')
subdir('pg_prewarm')
diff --git a/contrib/pg_copy_json/.gitignore b/contrib/pg_copy_json/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/contrib/pg_copy_json/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/contrib/pg_copy_json/Makefile b/contrib/pg_copy_json/Makefile
new file mode 100644
index 0000000000..b0a348d618
--- /dev/null
+++ b/contrib/pg_copy_json/Makefile
@@ -0,0 +1,23 @@
+# contrib/pg_copy_json//Makefile
+
+MODULE_big = pg_copy_json
+OBJS = \
+ $(WIN32RES) \
+ pg_copy_json.o
+PGFILEDESC = "pg_copy_json - COPY TO JSON (JavaScript Object Notation) format"
+
+EXTENSION = pg_copy_json
+DATA = pg_copy_json--1.0.sql
+
+REGRESS = test_copy_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/pg_copy_json
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/pg_copy_json/expected/pg_copy_json.out b/contrib/pg_copy_json/expected/pg_copy_json.out
new file mode 100644
index 0000000000..73633c2303
--- /dev/null
+++ b/contrib/pg_copy_json/expected/pg_copy_json.out
@@ -0,0 +1,80 @@
+--
+-- COPY TO JSON
+--
+CREATE EXTENSION pg_copy_json;
+-- test copying in JSON format with various styles
+-- of embedded line ending characters
+create temp table copytest (
+ style text,
+ test text,
+ filler int);
+insert into copytest values('DOS',E'abc\r\ndef',1);
+insert into copytest values('Unix',E'abc\ndef',2);
+insert into copytest values('Mac',E'abc\rdef',3);
+insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4);
+copy copytest to stdout with (format 'json');
+{"style":"DOS","test":"abc\r\ndef","filler":1}
+{"style":"Unix","test":"abc\ndef","filler":2}
+{"style":"Mac","test":"abc\rdef","filler":3}
+{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+-- pg_copy_json do not support COPY FROM
+copy copytest from stdout with (format 'json');
+ERROR: cannot use JSON mode in COPY FROM
+-- test copying in JSON format with various styles
+-- of embedded escaped characters
+create temp table copyjsontest (
+ id bigserial,
+ f1 text,
+ f2 timestamptz);
+insert into copyjsontest
+ select g.i,
+ CASE WHEN g.i % 2 = 0 THEN
+ 'line with '' in it: ' || g.i::text
+ ELSE
+ 'line with " in it: ' || g.i::text
+ END,
+ 'Mon Feb 10 17:32:01 1997 PST'
+ from generate_series(1,5) as g(i);
+insert into copyjsontest (f1) values
+(E'aaa\"bbb'::text),
+(E'aaa\\bbb'::text),
+(E'aaa\/bbb'::text),
+(E'aaa\bbbb'::text),
+(E'aaa\fbbb'::text),
+(E'aaa\nbbb'::text),
+(E'aaa\rbbb'::text),
+(E'aaa\tbbb'::text);
+copy copyjsontest to stdout with (format 'json');
+{"id":1,"f1":"line with \" in it: 1","f2":"1997-02-10T17:32:01-08:00"}
+{"id":2,"f1":"line with ' in it: 2","f2":"1997-02-10T17:32:01-08:00"}
+{"id":3,"f1":"line with \" in it: 3","f2":"1997-02-10T17:32:01-08:00"}
+{"id":4,"f1":"line with ' in it: 4","f2":"1997-02-10T17:32:01-08:00"}
+{"id":5,"f1":"line with \" in it: 5","f2":"1997-02-10T17:32:01-08:00"}
+{"id":1,"f1":"aaa\"bbb","f2":null}
+{"id":2,"f1":"aaa\\bbb","f2":null}
+{"id":3,"f1":"aaa/bbb","f2":null}
+{"id":4,"f1":"aaa\bbbb","f2":null}
+{"id":5,"f1":"aaa\fbbb","f2":null}
+{"id":6,"f1":"aaa\nbbb","f2":null}
+{"id":7,"f1":"aaa\rbbb","f2":null}
+{"id":8,"f1":"aaa\tbbb","f2":null}
+-- test force array
+copy copytest to stdout (format 'json', force_array);
+[
+ {"style":"DOS","test":"abc\r\ndef","filler":1}
+,{"style":"Unix","test":"abc\ndef","filler":2}
+,{"style":"Mac","test":"abc\rdef","filler":3}
+,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+]
+copy copytest to stdout (format 'json', force_array true);
+[
+ {"style":"DOS","test":"abc\r\ndef","filler":1}
+,{"style":"Unix","test":"abc\ndef","filler":2}
+,{"style":"Mac","test":"abc\rdef","filler":3}
+,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+]
+copy copytest to stdout (format 'json', force_array false);
+{"style":"DOS","test":"abc\r\ndef","filler":1}
+{"style":"Unix","test":"abc\ndef","filler":2}
+{"style":"Mac","test":"abc\rdef","filler":3}
+{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
diff --git a/contrib/pg_copy_json/meson.build b/contrib/pg_copy_json/meson.build
new file mode 100644
index 0000000000..71f9338267
--- /dev/null
+++ b/contrib/pg_copy_json/meson.build
@@ -0,0 +1,34 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+pg_copy_json_sources = files(
+ 'pg_copy_json.c',
+)
+
+if host_system == 'windows'
+ pg_copy_json_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'pg_copy_json',
+ '--FILEDESC', 'pg_copy_json - COPY TO JSON format',])
+endif
+
+pg_copy_json = shared_module('pg_copy_json',
+ pg_copy_json_sources,
+ kwargs: contrib_mod_args,
+)
+contrib_targets += pg_copy_json
+
+install_data(
+ 'pg_copy_json--1.0.sql',
+ 'pg_copy_json.control',
+ kwargs: contrib_data_args,
+)
+
+tests += {
+ 'name': 'pg_copy_json',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'pg_copy_json',
+ ],
+ },
+}
diff --git a/contrib/pg_copy_json/pg_copy_json--1.0.sql b/contrib/pg_copy_json/pg_copy_json--1.0.sql
new file mode 100644
index 0000000000..d738a1e7e9
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json--1.0.sql
@@ -0,0 +1,9 @@
+/* contrib/pg_copy_json/copy_json--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION pg_copy_json" to load this file. \quit
+
+CREATE FUNCTION pg_catalog.json(internal)
+ RETURNS copy_handler
+ AS 'MODULE_PATHNAME', 'copy_json'
+ LANGUAGE C;
diff --git a/contrib/pg_copy_json/pg_copy_json.c b/contrib/pg_copy_json/pg_copy_json.c
new file mode 100644
index 0000000000..cbfdee8e8b
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json.c
@@ -0,0 +1,218 @@
+/*--------------------------------------------------------------------------
+ *
+ * pg_copy_json.c
+ * COPY TO JSON (JavaScript Object Notation) format.
+ *
+ * Portions Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * contrib/test_copy_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/defrem.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "libpq/pqformat.h"
+#include "utils/json.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct
+{
+ /*
+ * Force output of square brackets as array decorations at the beginning
+ * and end of output, with commas between the rows.
+ */
+ bool force_array;
+ bool force_array_specified;
+
+ /* need delimiter to start next json array element */
+ bool json_row_delim_needed;
+} CopyJsonData;
+
+static inline void
+InitCopyJsonData(CopyJsonData *p)
+{
+ Assert(p);
+ p->force_array = false;
+ p->force_array_specified = false;
+ p->json_row_delim_needed = false;
+}
+
+static void
+CopyToJsonSendEndOfRow(CopyToState cstate)
+{
+ switch (cstate->copy_dest)
+ {
+ case COPY_DEST_FILE:
+ /* Default line termination depends on platform */
+#ifndef WIN32
+ CopySendChar(cstate, '\n');
+#else
+ CopySendString(cstate, "\r\n");
+#endif
+ break;
+ case COPY_DEST_FRONTEND:
+ /* The FE/BE protocol uses \n as newline for all platforms */
+ CopySendChar(cstate, '\n');
+ break;
+ default:
+ break;
+ }
+ CopyToStateFlush(cstate);
+}
+
+static bool
+CopyToJsonProcessOption(CopyToState cstate, DefElem *defel)
+{
+ CopyJsonData *p;
+
+ if (cstate->opaque == NULL)
+ {
+ MemoryContext oldcontext;
+ oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ cstate->opaque = palloc0(sizeof(CopyJsonData));
+ MemoryContextSwitchTo(oldcontext);
+ InitCopyJsonData(cstate->opaque);
+ }
+
+ p = (CopyJsonData *)cstate->opaque;
+
+ if (strcmp(defel->defname, "force_array") == 0)
+ {
+ if (p->force_array_specified)
+ ereport(ERROR,
+ errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("CopyToJsonProcessOption: redundant options \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel)));
+ p->force_array_specified = true;
+ p->force_array = defGetBoolean(defel);
+
+ return true;
+ }
+
+ return false;
+}
+
+static void
+CopyToJsonSendCopyBegin(CopyToState cstate)
+{
+ StringInfoData buf;
+ int16 format = 0;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ /*
+ * JSON mode is always one non-binary column
+ */
+ pq_sendint16(&buf, 1);
+ pq_sendint16(&buf, 0);
+ pq_endmessage(&buf);
+}
+
+static void
+CopyToJsonStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ CopyJsonData *p;
+
+ if (cstate->opaque == NULL)
+ {
+ MemoryContext oldcontext;
+ oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ cstate->opaque = palloc0(sizeof(CopyJsonData));
+ MemoryContextSwitchTo(oldcontext);
+ InitCopyJsonData(cstate->opaque);
+ }
+
+ /* No need to alloc cstate->out_functions */
+
+ p = (CopyJsonData *)cstate->opaque;
+
+ /* If FORCE_ARRAY has been specified send the open bracket. */
+ if (p->force_array)
+ {
+ CopySendChar(cstate, '[');
+ CopyToJsonSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToJsonOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ Datum rowdata;
+ StringInfo result;
+ CopyJsonData *p;
+
+ Assert(cstate->opaque);
+ p = (CopyJsonData *)cstate->opaque;
+
+ if(!cstate->rel)
+ {
+ for (int i = 0; i < slot->tts_tupleDescriptor->natts; i++)
+ {
+ /* Flat-copy the attribute array */
+ memcpy(TupleDescAttr(slot->tts_tupleDescriptor, i),
+ TupleDescAttr(cstate->queryDesc->tupDesc, i),
+ 1 * sizeof(FormData_pg_attribute));
+ }
+ BlessTupleDesc(slot->tts_tupleDescriptor);
+ }
+ rowdata = ExecFetchSlotHeapTupleDatum(slot);
+ result = makeStringInfo();
+ composite_to_json(rowdata, result, false);
+
+ if (p->json_row_delim_needed)
+ CopySendChar(cstate, ',');
+ else if (p->force_array)
+ {
+ /* first row needs no delimiter */
+ CopySendChar(cstate, ' ');
+ p->json_row_delim_needed = true;
+ }
+ CopySendData(cstate, result->data, result->len);
+ CopyToJsonSendEndOfRow(cstate);
+}
+
+static void
+CopyToJsonEnd(CopyToState cstate)
+{
+ CopyJsonData *p;
+
+ Assert(cstate->opaque);
+ p = (CopyJsonData *)cstate->opaque;
+
+ /* If FORCE_ARRAY has been specified send the close bracket. */
+ if (p->force_array)
+ {
+ CopySendChar(cstate, ']');
+ CopyToJsonSendEndOfRow(cstate);
+ }
+}
+
+static const CopyToRoutine CopyToRoutineJson = {
+ .type = T_CopyToRoutine,
+ .CopyToProcessOption = CopyToJsonProcessOption,
+ .CopyToSendCopyBegin = CopyToJsonSendCopyBegin,
+ .CopyToStart = CopyToJsonStart,
+ .CopyToOneRow = CopyToJsonOneRow,
+ .CopyToEnd = CopyToJsonEnd,
+};
+
+PG_FUNCTION_INFO_V1(copy_json);
+Datum
+copy_json(PG_FUNCTION_ARGS)
+{
+ bool is_from = PG_GETARG_BOOL(0);
+
+ if (is_from)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use JSON mode in COPY FROM")));
+
+ PG_RETURN_POINTER(&CopyToRoutineJson);
+}
diff --git a/contrib/pg_copy_json/pg_copy_json.control b/contrib/pg_copy_json/pg_copy_json.control
new file mode 100644
index 0000000000..90b0a74603
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json.control
@@ -0,0 +1,5 @@
+# pg_copy_json extension
+comment = 'COPY TO JSON format'
+default_version = '1.0'
+module_pathname = '$libdir/pg_copy_json'
+relocatable = true
diff --git a/contrib/pg_copy_json/sql/pg_copy_json.sql b/contrib/pg_copy_json/sql/pg_copy_json.sql
new file mode 100644
index 0000000000..73e7e514ac
--- /dev/null
+++ b/contrib/pg_copy_json/sql/pg_copy_json.sql
@@ -0,0 +1,59 @@
+--
+-- COPY TO JSON
+--
+
+CREATE EXTENSION pg_copy_json;
+
+-- test copying in JSON format with various styles
+-- of embedded line ending characters
+
+create temp table copytest (
+ style text,
+ test text,
+ filler int);
+
+insert into copytest values('DOS',E'abc\r\ndef',1);
+insert into copytest values('Unix',E'abc\ndef',2);
+insert into copytest values('Mac',E'abc\rdef',3);
+insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4);
+
+copy copytest to stdout with (format 'json');
+
+-- pg_copy_json do not support COPY FROM
+copy copytest from stdout with (format 'json');
+
+-- test copying in JSON format with various styles
+-- of embedded escaped characters
+
+create temp table copyjsontest (
+ id bigserial,
+ f1 text,
+ f2 timestamptz);
+
+insert into copyjsontest
+ select g.i,
+ CASE WHEN g.i % 2 = 0 THEN
+ 'line with '' in it: ' || g.i::text
+ ELSE
+ 'line with " in it: ' || g.i::text
+ END,
+ 'Mon Feb 10 17:32:01 1997 PST'
+ from generate_series(1,5) as g(i);
+
+insert into copyjsontest (f1) values
+(E'aaa\"bbb'::text),
+(E'aaa\\bbb'::text),
+(E'aaa\/bbb'::text),
+(E'aaa\bbbb'::text),
+(E'aaa\fbbb'::text),
+(E'aaa\nbbb'::text),
+(E'aaa\rbbb'::text),
+(E'aaa\tbbb'::text);
+
+copy copyjsontest to stdout with (format 'json');
+
+-- test force array
+
+copy copytest to stdout (format 'json', force_array);
+copy copytest to stdout (format 'json', force_array true);
+copy copytest to stdout (format 'json', force_array false);
diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c
index d719a61f16..fabd4e611e 100644
--- a/src/backend/utils/adt/json.c
+++ b/src/backend/utils/adt/json.c
@@ -83,8 +83,6 @@ typedef struct JsonAggState
JsonUniqueBuilderState unique_check;
} JsonAggState;
-static void composite_to_json(Datum composite, StringInfo result,
- bool use_line_feeds);
static void array_dim_to_json(StringInfo result, int dim, int ndims, int *dims,
Datum *vals, bool *nulls, int *valcount,
JsonTypeCategory tcategory, Oid outfuncoid,
@@ -507,8 +505,9 @@ array_to_json_internal(Datum array, StringInfo result, bool use_line_feeds)
/*
* Turn a composite / record into JSON.
+ * Exported so COPY TO can use it.
*/
-static void
+void
composite_to_json(Datum composite, StringInfo result, bool use_line_feeds)
{
HeapTupleHeader td;
diff --git a/src/include/utils/json.h b/src/include/utils/json.h
index 6d7f1b387d..d5631171ad 100644
--- a/src/include/utils/json.h
+++ b/src/include/utils/json.h
@@ -17,6 +17,8 @@
#include "lib/stringinfo.h"
/* functions in json.c */
+extern void composite_to_json(Datum composite, StringInfo result,
+ bool use_line_feeds);
extern void escape_json(StringInfo buf, const char *str);
extern char *JsonEncodeDateTime(char *buf, Datum value, Oid typid,
const int *tzp);
--
2.41.0
[application/octet-stream] v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch (4.5K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/10-v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch)
download | inline diff:
From 3e847de1acb2fd6966ef01192204448711ca3d5e Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 14:19:08 +0900
Subject: [PATCH v8 08/10] Add support for implementing custom COPY FROM format
as extension
* Add CopyFromStateData::opaque that can be used to keep data for
custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
method for CopyFromState and "BinaryData" is redundant.
---
src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
src/include/commands/copyapi.h | 5 +++++
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index a78a790060..f8a194635d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
static void CopyLoadInputBuf(CopyFromState cstate);
-static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
ReceiveCopyBegin(CopyFromState cstate)
@@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
int32 tmp;
/* Signature */
- if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+ if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
memcmp(readSig, BinarySignature, 11) != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
/* Skip extension header, if present */
while (tmp-- > 0)
{
- if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+ if (CopyFromStateRead(cstate, readSig, 1) != 1)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("invalid COPY file header (wrong length)")));
@@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
{
uint32 buf;
- if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+ if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
{
*val = 0; /* suppress compiler warning */
return false;
@@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
{
uint16 buf;
- if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+ if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
{
*val = 0; /* suppress compiler warning */
return false;
@@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate)
}
/*
- * CopyReadBinaryData
+ * CopyFromStateRead
*
* Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
* and writes them to 'dest'. Returns the number of bytes read (which
* would be less than 'nbytes' only if we reach EOF).
*/
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
{
int copied_bytes = 0;
@@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
*/
char dummy;
- if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyFromStateRead(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
resetStringInfo(&cstate->attribute_buf);
enlargeStringInfo(&cstate->attribute_buf, fld_size);
- if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
- fld_size) != fld_size)
+ if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+ fld_size) != fld_size)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unexpected EOF in COPY data")));
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b7e8f627bf..22accc83ab 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -314,8 +314,13 @@ typedef struct CopyFromStateData
#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* For custom format implementation */
+ void *opaque; /* private space */
} CopyFromStateData;
+extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
/*
* Represents the different dest cases we need to worry about at
* the bottom level
--
2.41.0
[application/octet-stream] v8-0007-Export-CopyFromStateData.patch (17.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/11-v8-0007-Export-CopyFromStateData.patch)
download | inline diff:
From 1ed575fda7f196ea411e9e53dd9c0739f160fb78 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 14:16:29 +0900
Subject: [PATCH v8 07/10] Export CopyFromStateData
It's for custom COPY FROM format handlers implemented as extension.
This just moves codes. This doesn't change codes except CopySource
enum values. CopySource enum values changes aren't required but I did
like I did for CopyDest enum values. I changed COPY_ prefix to
COPY_SOURCE_ prefix. For example, COPY_FILE to COPY_SOURCE_FILE.
Note that this change isn't enough to implement a custom COPY FROM
format handler as extension. We'll do the followings in a subsequent
commit:
1. Add an opaque space for custom COPY FROM format handler
2. Export CopyReadBinaryData() to read the next data
---
src/backend/commands/copyfrom.c | 4 +-
src/backend/commands/copyfromparse.c | 10 +-
src/include/commands/copy.h | 2 -
src/include/commands/copyapi.h | 156 ++++++++++++++++++++++-
src/include/commands/copyfrom_internal.h | 150 ----------------------
5 files changed, 162 insertions(+), 160 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index d556ebb5d6..b4ac7cbd2c 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1710,7 +1710,7 @@ BeginCopyFrom(ParseState *pstate,
pg_encoding_to_char(GetDatabaseEncoding()))));
}
- cstate->copy_src = COPY_FILE; /* default */
+ cstate->copy_src = COPY_SOURCE_FILE; /* default */
cstate->whereClause = whereClause;
@@ -1829,7 +1829,7 @@ BeginCopyFrom(ParseState *pstate,
if (data_source_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_src = COPY_CALLBACK;
+ cstate->copy_src = COPY_SOURCE_CALLBACK;
cstate->data_source_cb = data_source_cb;
}
else if (pipe)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 49632f75e4..a78a790060 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -181,7 +181,7 @@ ReceiveCopyBegin(CopyFromState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_src = COPY_FRONTEND;
+ cstate->copy_src = COPY_SOURCE_FRONTEND;
cstate->fe_msgbuf = makeStringInfo();
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -249,7 +249,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
switch (cstate->copy_src)
{
- case COPY_FILE:
+ case COPY_SOURCE_FILE:
bytesread = fread(databuf, 1, maxread, cstate->copy_file);
if (ferror(cstate->copy_file))
ereport(ERROR,
@@ -258,7 +258,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
if (bytesread == 0)
cstate->raw_reached_eof = true;
break;
- case COPY_FRONTEND:
+ case COPY_SOURCE_FRONTEND:
while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
{
int avail;
@@ -341,7 +341,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
bytesread += avail;
}
break;
- case COPY_CALLBACK:
+ case COPY_SOURCE_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
break;
}
@@ -1099,7 +1099,7 @@ CopyReadLine(CopyFromState cstate)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (cstate->copy_src == COPY_SOURCE_FRONTEND)
{
int inbytes;
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index df29d42555..cd41d32074 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,8 +20,6 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
-
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
uint64 *processed);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index ef1bb201c2..b7e8f627bf 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,11 +14,12 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "commands/trigger.h"
#include "executor/execdesc.h"
#include "executor/tuptable.h"
+#include "nodes/miscnodes.h"
#include "nodes/parsenodes.h"
-/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
@@ -162,6 +163,159 @@ typedef struct CopyFormatOptions
CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
+
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_SOURCE_FILE, /* from file (or a piped program) */
+ COPY_SOURCE_FRONTEND, /* from frontend */
+ COPY_SOURCE_CALLBACK, /* from callback function */
+} CopySource;
+
+/*
+ * Represents the end-of-line terminator type of the input
+ */
+typedef enum EolType
+{
+ EOL_UNKNOWN,
+ EOL_NL,
+ EOL_CR,
+ EOL_CRNL,
+} EolType;
+
+typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
+
+/*
+ * This struct contains all the state variables used throughout a COPY FROM
+ * operation.
+ */
+typedef struct CopyFromStateData
+{
+ /* low-level state data */
+ CopySource copy_src; /* type of copy source */
+ FILE *copy_file; /* used if copy_src == COPY_FILE */
+ StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
+
+ EolType eol_type; /* EOL type of input */
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ Oid conversion_proc; /* encoding conversion function */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ CopyFormatOptions opts;
+ bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /* these are just for error messages, see CopyFromErrorCallback */
+ const char *cur_relname; /* table name for error messages */
+ uint64 cur_lineno; /* line number for error messages */
+ const char *cur_attname; /* current att for error messages */
+ const char *cur_attval; /* current att value for error messages */
+ bool relname_only; /* don't output line number, att, etc. */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ AttrNumber num_defaults; /* count of att that are missing and have
+ * default value */
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+ ErrorSaveContext *escontext; /* soft error trapper during in_functions
+ * execution */
+ uint64 num_errors; /* total number of rows which contained soft
+ * errors */
+ int *defmap; /* array of default att numbers related to
+ * missing att */
+ ExprState **defexprs; /* array of default att expressions for all
+ * att */
+ bool *defaults; /* if DEFAULT marker was found for
+ * corresponding att */
+ bool volatile_defexprs; /* is any of defexprs volatile? */
+ List *range_table; /* single element list of RangeTblEntry */
+ List *rteperminfos; /* single element list of RTEPermissionInfo */
+ ExprState *qualexpr;
+
+ TransitionCaptureState *transition_capture;
+
+ /*
+ * These variables are used to reduce overhead in COPY FROM.
+ *
+ * attribute_buf holds the separated, de-escaped text for each field of
+ * the current line. The CopyReadAttributes functions return arrays of
+ * pointers into this buffer. We avoid palloc/pfree overhead by re-using
+ * the buffer on each cycle.
+ *
+ * In binary COPY FROM, attribute_buf holds the binary data for the
+ * current field, but the usage is otherwise similar.
+ */
+ StringInfoData attribute_buf;
+
+ /* field raw data pointers found by COPY FROM */
+
+ int max_fields;
+ char **raw_fields;
+
+ /*
+ * Similarly, line_buf holds the whole input line being processed. The
+ * input cycle is first to read the whole line into line_buf, and then
+ * extract the individual attribute fields into attribute_buf. line_buf
+ * is preserved unmodified so that we can display it in error messages if
+ * appropriate. (In binary mode, line_buf is not used.)
+ */
+ StringInfoData line_buf;
+ bool line_buf_valid; /* contains the row being processed? */
+
+ /*
+ * input_buf holds input data, already converted to database encoding.
+ *
+ * In text mode, CopyReadLine parses this data sufficiently to locate line
+ * boundaries, then transfers the data to line_buf. We guarantee that
+ * there is a \0 at input_buf[input_buf_len] at all times. (In binary
+ * mode, input_buf is not used.)
+ *
+ * If encoding conversion is not required, input_buf is not a separate
+ * buffer but points directly to raw_buf. In that case, input_buf_len
+ * tracks the number of bytes that have been verified as valid in the
+ * database encoding, and raw_buf_len is the total number of bytes stored
+ * in the buffer.
+ */
+#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
+ char *input_buf;
+ int input_buf_index; /* next byte to process */
+ int input_buf_len; /* total # of bytes stored */
+ bool input_reached_eof; /* true if we reached EOF */
+ bool input_reached_error; /* true if a conversion error happened */
+ /* Shorthand for number of unconsumed bytes available in input_buf */
+#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
+
+ /*
+ * raw_buf holds raw input data read from the data source (file or client
+ * connection), not yet converted to the database encoding. Like with
+ * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
+ */
+#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
+ char *raw_buf;
+ int raw_buf_index; /* next byte to process */
+ int raw_buf_len; /* total # of bytes stored */
+ bool raw_reached_eof; /* true if we reached EOF */
+
+ /* Shorthand for number of unconsumed bytes available in raw_buf */
+#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
+
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyFromStateData;
+
/*
* Represents the different dest cases we need to worry about at
* the bottom level
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 921c1513f7..f8f6120255 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -18,28 +18,6 @@
#include "commands/trigger.h"
#include "nodes/miscnodes.h"
-/*
- * Represents the different source cases we need to worry about at
- * the bottom level
- */
-typedef enum CopySource
-{
- COPY_FILE, /* from file (or a piped program) */
- COPY_FRONTEND, /* from frontend */
- COPY_CALLBACK, /* from callback function */
-} CopySource;
-
-/*
- * Represents the end-of-line terminator type of the input
- */
-typedef enum EolType
-{
- EOL_UNKNOWN,
- EOL_NL,
- EOL_CR,
- EOL_CRNL,
-} EolType;
-
/*
* Represents the insert method to be used during COPY FROM.
*/
@@ -52,134 +30,6 @@ typedef enum CopyInsertMethod
* ExecForeignBatchInsert only if valid */
} CopyInsertMethod;
-/*
- * This struct contains all the state variables used throughout a COPY FROM
- * operation.
- */
-typedef struct CopyFromStateData
-{
- /* low-level state data */
- CopySource copy_src; /* type of copy source */
- FILE *copy_file; /* used if copy_src == COPY_FILE */
- StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
-
- EolType eol_type; /* EOL type of input */
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- Oid conversion_proc; /* encoding conversion function */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
- bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /* these are just for error messages, see CopyFromErrorCallback */
- const char *cur_relname; /* table name for error messages */
- uint64 cur_lineno; /* line number for error messages */
- const char *cur_attname; /* current att for error messages */
- const char *cur_attval; /* current att value for error messages */
- bool relname_only; /* don't output line number, att, etc. */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- AttrNumber num_defaults; /* count of att that are missing and have
- * default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
- ErrorSaveContext *escontext; /* soft error trapper during in_functions
- * execution */
- uint64 num_errors; /* total number of rows which contained soft
- * errors */
- int *defmap; /* array of default att numbers related to
- * missing att */
- ExprState **defexprs; /* array of default att expressions for all
- * att */
- bool *defaults; /* if DEFAULT marker was found for
- * corresponding att */
- bool volatile_defexprs; /* is any of defexprs volatile? */
- List *range_table; /* single element list of RangeTblEntry */
- List *rteperminfos; /* single element list of RTEPermissionInfo */
- ExprState *qualexpr;
-
- TransitionCaptureState *transition_capture;
-
- /*
- * These variables are used to reduce overhead in COPY FROM.
- *
- * attribute_buf holds the separated, de-escaped text for each field of
- * the current line. The CopyReadAttributes functions return arrays of
- * pointers into this buffer. We avoid palloc/pfree overhead by re-using
- * the buffer on each cycle.
- *
- * In binary COPY FROM, attribute_buf holds the binary data for the
- * current field, but the usage is otherwise similar.
- */
- StringInfoData attribute_buf;
-
- /* field raw data pointers found by COPY FROM */
-
- int max_fields;
- char **raw_fields;
-
- /*
- * Similarly, line_buf holds the whole input line being processed. The
- * input cycle is first to read the whole line into line_buf, and then
- * extract the individual attribute fields into attribute_buf. line_buf
- * is preserved unmodified so that we can display it in error messages if
- * appropriate. (In binary mode, line_buf is not used.)
- */
- StringInfoData line_buf;
- bool line_buf_valid; /* contains the row being processed? */
-
- /*
- * input_buf holds input data, already converted to database encoding.
- *
- * In text mode, CopyReadLine parses this data sufficiently to locate line
- * boundaries, then transfers the data to line_buf. We guarantee that
- * there is a \0 at input_buf[input_buf_len] at all times. (In binary
- * mode, input_buf is not used.)
- *
- * If encoding conversion is not required, input_buf is not a separate
- * buffer but points directly to raw_buf. In that case, input_buf_len
- * tracks the number of bytes that have been verified as valid in the
- * database encoding, and raw_buf_len is the total number of bytes stored
- * in the buffer.
- */
-#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
- char *input_buf;
- int input_buf_index; /* next byte to process */
- int input_buf_len; /* total # of bytes stored */
- bool input_reached_eof; /* true if we reached EOF */
- bool input_reached_error; /* true if a conversion error happened */
- /* Shorthand for number of unconsumed bytes available in input_buf */
-#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
-
- /*
- * raw_buf holds raw input data read from the data source (file or client
- * connection), not yet converted to the database encoding. Like with
- * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
- */
-#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
- char *raw_buf;
- int raw_buf_index; /* next byte to process */
- int raw_buf_len; /* total # of bytes stored */
- bool raw_reached_eof; /* true if we reached EOF */
-
- /* Shorthand for number of unconsumed bytes available in raw_buf */
-#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
-
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
-
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
--
2.41.0
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 06:03 ` Sutou Kouhei <[email protected]>
2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-29 06:03 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800,
Junwang Zhao <[email protected]> wrote:
> I have been working on a *COPY TO JSON* extension since yesterday,
> which is based on your V6 patch set, I'd like to give you more input
> so you can make better decisions about the implementation(with only
> pg-copy-arrow you might not get everything considered).
Thanks!
> 0009 is some changes made by me, I changed CopyToGetFormat to
> CopyToSendCopyBegin because pg_copy_json need to send different bytes
> in SendCopyBegin, get the format code along is not enough
Oh, I haven't cared about the case.
How about the following API instead?
static void
SendCopyBegin(CopyToState cstate)
{
StringInfoData buf;
pq_beginmessage(&buf, PqMsg_CopyOutResponse);
cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf);
pq_endmessage(&buf);
cstate->copy_dest = COPY_FRONTEND;
}
static void
CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf)
{
int16 format = 0;
pq_sendbyte(&buf, format); /* overall format */
/*
* JSON mode is always one non-binary column
*/
pq_sendint16(&buf, 1);
pq_sendint16(&buf, format);
}
> 00010 is the pg_copy_json extension, I think this should be a good
> case which can utilize the *extendable copy format* feature
It seems that it's convenient that we have one more callback
for initializing CopyToState::opaque. It's called only once
when Copy{To,From}Routine is chosen:
typedef struct CopyToRoutine
{
void (*CopyToInit) (CopyToState cstate);
...
};
void
ProcessCopyOptions(ParseState *pstate,
CopyFormatOptions *opts_out,
bool is_from,
void *cstate,
List *options)
{
...
foreach(option, options)
{
DefElem *defel = lfirst_node(DefElem, option);
if (strcmp(defel->defname, "format") == 0)
{
...
opts_out->to_routine = &CopyToRoutineXXX;
opts_out->to_routine->CopyToInit(cstate);
...
}
}
...
}
> maybe we
> should delete copy_test_format if we have this extension as an
> example?
I haven't read the COPY TO format json thread[1] carefully
(sorry), but we may add the JSON format as a built-in
format. If we do it, copy_test_format is useful to test the
extension API.
[1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m...
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-29 06:48 ` Junwang Zhao <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Junwang Zhao @ 2024-01-29 06:48 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 2:03 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > I have been working on a *COPY TO JSON* extension since yesterday,
> > which is based on your V6 patch set, I'd like to give you more input
> > so you can make better decisions about the implementation(with only
> > pg-copy-arrow you might not get everything considered).
>
> Thanks!
>
> > 0009 is some changes made by me, I changed CopyToGetFormat to
> > CopyToSendCopyBegin because pg_copy_json need to send different bytes
> > in SendCopyBegin, get the format code along is not enough
>
> Oh, I haven't cared about the case.
> How about the following API instead?
>
> static void
> SendCopyBegin(CopyToState cstate)
> {
> StringInfoData buf;
>
> pq_beginmessage(&buf, PqMsg_CopyOutResponse);
> cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf);
> pq_endmessage(&buf);
> cstate->copy_dest = COPY_FRONTEND;
> }
>
> static void
> CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf)
> {
> int16 format = 0;
>
> pq_sendbyte(&buf, format); /* overall format */
> /*
> * JSON mode is always one non-binary column
> */
> pq_sendint16(&buf, 1);
> pq_sendint16(&buf, format);
> }
Make sense to me.
>
> > 00010 is the pg_copy_json extension, I think this should be a good
> > case which can utilize the *extendable copy format* feature
>
> It seems that it's convenient that we have one more callback
> for initializing CopyToState::opaque. It's called only once
> when Copy{To,From}Routine is chosen:
>
> typedef struct CopyToRoutine
> {
> void (*CopyToInit) (CopyToState cstate);
> ...
> };
I like this, we can alloc private data in this hook.
>
> void
> ProcessCopyOptions(ParseState *pstate,
> CopyFormatOptions *opts_out,
> bool is_from,
> void *cstate,
> List *options)
> {
> ...
> foreach(option, options)
> {
> DefElem *defel = lfirst_node(DefElem, option);
>
> if (strcmp(defel->defname, "format") == 0)
> {
> ...
> opts_out->to_routine = &CopyToRoutineXXX;
> opts_out->to_routine->CopyToInit(cstate);
> ...
> }
> }
> ...
> }
>
>
> > maybe we
> > should delete copy_test_format if we have this extension as an
> > example?
>
> I haven't read the COPY TO format json thread[1] carefully
> (sorry), but we may add the JSON format as a built-in
> format. If we do it, copy_test_format is useful to test the
> extension API.
Yeah, maybe, I have no strong opinion here, pg_copy_json is
just a toy extension for discussion.
>
> [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m...
>
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 02:41 ` Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2024-01-29 02:41 UTC (permalink / raw)
To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > Junwang Zhao <[email protected]> wrote:
> >
> > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > single option, and store the options in the opaque field, but it can not
> > > check the relation of two options, for example, considering json format,
> > > the `header` option can not be handled by these two functions.
> > >
> > > I want to find a way when the user specifies the header option, customer
> > > handler can error out.
> >
> > Ah, you want to use a built-in option (such as "header")
> > value from a custom handler, right? Hmm, it may be better
> > that we call CopyToProcessOption()/CopyFromProcessOption()
> > for all options including built-in options.
> >
> Hmm, still I don't think it can handle all cases, since we don't know
> the sequence of the options, we need all the options been parsed
> before we check the compatibility of the options, or customer
> handlers will need complicated logic to resolve that, which might
> lead to ugly code :(
>
Does it make sense to pass only non-builtin options to the custom
format callback after parsing and evaluating the builtin options? That
is, we parse and evaluate only the builtin options and populate
opts_out first, then pass each rest option to the custom format
handler callback. The callback can refer to the builtin option values.
The callback is expected to return false if the passed option is not
supported. If one of the builtin formats is specified and the rest
options list has at least one option, we raise "option %s not
recognized" error. IOW it's the core's responsibility to ranse the
"option %s not recognized" error, which is in order to raise a
consistent error message. Also, I think the core should check the
redundant options including bultiin and custom options.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-29 03:10 ` Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Junwang Zhao @ 2024-01-29 03:10 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> >
> > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > >
> > > Hi,
> > >
> > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > Junwang Zhao <[email protected]> wrote:
> > >
> > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > single option, and store the options in the opaque field, but it can not
> > > > check the relation of two options, for example, considering json format,
> > > > the `header` option can not be handled by these two functions.
> > > >
> > > > I want to find a way when the user specifies the header option, customer
> > > > handler can error out.
> > >
> > > Ah, you want to use a built-in option (such as "header")
> > > value from a custom handler, right? Hmm, it may be better
> > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > for all options including built-in options.
> > >
> > Hmm, still I don't think it can handle all cases, since we don't know
> > the sequence of the options, we need all the options been parsed
> > before we check the compatibility of the options, or customer
> > handlers will need complicated logic to resolve that, which might
> > lead to ugly code :(
> >
>
> Does it make sense to pass only non-builtin options to the custom
> format callback after parsing and evaluating the builtin options? That
> is, we parse and evaluate only the builtin options and populate
> opts_out first, then pass each rest option to the custom format
> handler callback. The callback can refer to the builtin option values.
Yeah, I think this makes sense.
> The callback is expected to return false if the passed option is not
> supported. If one of the builtin formats is specified and the rest
> options list has at least one option, we raise "option %s not
> recognized" error. IOW it's the core's responsibility to ranse the
> "option %s not recognized" error, which is in order to raise a
> consistent error message. Also, I think the core should check the
> redundant options including bultiin and custom options.
It would be good that core could check all the redundant options,
but where should core do the book-keeping of all the options? I have
no idea about this, in my implementation of pg_copy_json extension,
I handle redundant options by adding a xxx_specified field for each
xxx.
>
> Regards,
>
> --
> Masahiko Sawada
> Amazon Web Services: https://aws.amazon.com
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 03:21 ` Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2024-01-29 03:21 UTC (permalink / raw)
To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote:
>
> On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
> >
> > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> > >
> > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > > >
> > > > Hi,
> > > >
> > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > > Junwang Zhao <[email protected]> wrote:
> > > >
> > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > > single option, and store the options in the opaque field, but it can not
> > > > > check the relation of two options, for example, considering json format,
> > > > > the `header` option can not be handled by these two functions.
> > > > >
> > > > > I want to find a way when the user specifies the header option, customer
> > > > > handler can error out.
> > > >
> > > > Ah, you want to use a built-in option (such as "header")
> > > > value from a custom handler, right? Hmm, it may be better
> > > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > > for all options including built-in options.
> > > >
> > > Hmm, still I don't think it can handle all cases, since we don't know
> > > the sequence of the options, we need all the options been parsed
> > > before we check the compatibility of the options, or customer
> > > handlers will need complicated logic to resolve that, which might
> > > lead to ugly code :(
> > >
> >
> > Does it make sense to pass only non-builtin options to the custom
> > format callback after parsing and evaluating the builtin options? That
> > is, we parse and evaluate only the builtin options and populate
> > opts_out first, then pass each rest option to the custom format
> > handler callback. The callback can refer to the builtin option values.
>
> Yeah, I think this makes sense.
>
> > The callback is expected to return false if the passed option is not
> > supported. If one of the builtin formats is specified and the rest
> > options list has at least one option, we raise "option %s not
> > recognized" error. IOW it's the core's responsibility to ranse the
> > "option %s not recognized" error, which is in order to raise a
> > consistent error message. Also, I think the core should check the
> > redundant options including bultiin and custom options.
>
> It would be good that core could check all the redundant options,
> but where should core do the book-keeping of all the options? I have
> no idea about this, in my implementation of pg_copy_json extension,
> I handle redundant options by adding a xxx_specified field for each
> xxx.
What I imagined is that while parsing the all specified options, we
evaluate builtin options and we add non-builtin options to another
list. Then when parsing a non-builtin option, we check if this option
already exists in the list. If there is, we raise the "option %s not
recognized" error.". Once we complete checking all options, we pass
each option in the list to the callback.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-29 03:37 ` Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Junwang Zhao @ 2024-01-29 03:37 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 11:22 AM Masahiko Sawada <[email protected]> wrote:
>
> On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote:
> >
> > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
> > >
> > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> > > >
> > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > > > >
> > > > > Hi,
> > > > >
> > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > > > Junwang Zhao <[email protected]> wrote:
> > > > >
> > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > > > single option, and store the options in the opaque field, but it can not
> > > > > > check the relation of two options, for example, considering json format,
> > > > > > the `header` option can not be handled by these two functions.
> > > > > >
> > > > > > I want to find a way when the user specifies the header option, customer
> > > > > > handler can error out.
> > > > >
> > > > > Ah, you want to use a built-in option (such as "header")
> > > > > value from a custom handler, right? Hmm, it may be better
> > > > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > > > for all options including built-in options.
> > > > >
> > > > Hmm, still I don't think it can handle all cases, since we don't know
> > > > the sequence of the options, we need all the options been parsed
> > > > before we check the compatibility of the options, or customer
> > > > handlers will need complicated logic to resolve that, which might
> > > > lead to ugly code :(
> > > >
> > >
> > > Does it make sense to pass only non-builtin options to the custom
> > > format callback after parsing and evaluating the builtin options? That
> > > is, we parse and evaluate only the builtin options and populate
> > > opts_out first, then pass each rest option to the custom format
> > > handler callback. The callback can refer to the builtin option values.
> >
> > Yeah, I think this makes sense.
> >
> > > The callback is expected to return false if the passed option is not
> > > supported. If one of the builtin formats is specified and the rest
> > > options list has at least one option, we raise "option %s not
> > > recognized" error. IOW it's the core's responsibility to ranse the
> > > "option %s not recognized" error, which is in order to raise a
> > > consistent error message. Also, I think the core should check the
> > > redundant options including bultiin and custom options.
> >
> > It would be good that core could check all the redundant options,
> > but where should core do the book-keeping of all the options? I have
> > no idea about this, in my implementation of pg_copy_json extension,
> > I handle redundant options by adding a xxx_specified field for each
> > xxx.
>
> What I imagined is that while parsing the all specified options, we
> evaluate builtin options and we add non-builtin options to another
> list. Then when parsing a non-builtin option, we check if this option
> already exists in the list. If there is, we raise the "option %s not
> recognized" error.". Once we complete checking all options, we pass
> each option in the list to the callback.
LGTM.
>
> Regards,
>
> --
> Masahiko Sawada
> Amazon Web Services: https://aws.amazon.com
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 09:45 ` Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-29 09:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800,
Junwang Zhao <[email protected]> wrote:
>> > > Does it make sense to pass only non-builtin options to the custom
>> > > format callback after parsing and evaluating the builtin options? That
>> > > is, we parse and evaluate only the builtin options and populate
>> > > opts_out first, then pass each rest option to the custom format
>> > > handler callback. The callback can refer to the builtin option values.
>>
>> What I imagined is that while parsing the all specified options, we
>> evaluate builtin options and we add non-builtin options to another
>> list. Then when parsing a non-builtin option, we check if this option
>> already exists in the list. If there is, we raise the "option %s not
>> recognized" error.". Once we complete checking all options, we pass
>> each option in the list to the callback.
I implemented this idea and the following ideas:
1. Add init callback for initialization
2. Change GetFormat() to FillCopyXXXResponse()
because JSON format always use 1 column
3. FROM only: Eliminate more cstate->opts.csv_mode branches
(This is for performance.)
See the attached v9 patch set for details. Changes since v7:
0001:
* Move CopyToProcessOption() calls to the end of
ProcessCopyOptions() for easy to option validation
* Add CopyToState::CopyToInit() and call it in
ProcessCopyOptionFormatTo()
* Change CopyToState::CopyToGetFormat() to
CopyToState::CopyToFillCopyOutResponse() and use it in
SendCopyBegin()
0002:
* Move CopyFromProcessOption() calls to the end of
ProcessCopyOptions() for easy to option validation
* Add CopyFromState::CopyFromInit() and call it in
ProcessCopyOptionFormatFrom()
* Change CopyFromState::CopyFromGetFormat() to
CopyFromState::CopyFromFillCopyOutResponse() and use it in
ReceiveCopyBegin()
* Rename NextCopyFromRawFields() to
NextCopyFromRawFieldsInternal() and pass the read
attributes callback explicitly to eliminate more
cstate->opts.csv_mode branches
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 02:11 ` Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2024-01-30 02:11 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 6:45 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> >> > > Does it make sense to pass only non-builtin options to the custom
> >> > > format callback after parsing and evaluating the builtin options? That
> >> > > is, we parse and evaluate only the builtin options and populate
> >> > > opts_out first, then pass each rest option to the custom format
> >> > > handler callback. The callback can refer to the builtin option values.
> >>
> >> What I imagined is that while parsing the all specified options, we
> >> evaluate builtin options and we add non-builtin options to another
> >> list. Then when parsing a non-builtin option, we check if this option
> >> already exists in the list. If there is, we raise the "option %s not
> >> recognized" error.". Once we complete checking all options, we pass
> >> each option in the list to the callback.
>
> I implemented this idea and the following ideas:
>
> 1. Add init callback for initialization
> 2. Change GetFormat() to FillCopyXXXResponse()
> because JSON format always use 1 column
> 3. FROM only: Eliminate more cstate->opts.csv_mode branches
> (This is for performance.)
>
> See the attached v9 patch set for details. Changes since v7:
>
> 0001:
>
> * Move CopyToProcessOption() calls to the end of
> ProcessCopyOptions() for easy to option validation
> * Add CopyToState::CopyToInit() and call it in
> ProcessCopyOptionFormatTo()
> * Change CopyToState::CopyToGetFormat() to
> CopyToState::CopyToFillCopyOutResponse() and use it in
> SendCopyBegin()
Thank you for updating the patch! Here are comments on 0001 patch:
---
+ if (!format_specified)
+ /* Set the default format. */
+ ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
+
I think we can pass "text" in this case instead of NULL. That way,
ProcessCopyOptionFormatTo doesn't need to handle NULL case.
We need curly brackets for this "if branch" as follows:
if (!format_specifed)
{
/* Set the default format. */
ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
}
---
+ /* Process not built-in options. */
+ foreach(option, unknown_options)
+ {
+ DefElem *defel = lfirst_node(DefElem, option);
+ bool processed = false;
+
+ if (!is_from)
+ processed =
opts_out->to_routine->CopyToProcessOption(cstate, defel);
+ if (!processed)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("option \"%s\" not recognized",
+ defel->defname),
+ parser_errposition(pstate,
defel->location)));
+ }
+ list_free(unknown_options);
I think we can check the duplicated options in the core as we discussed.
---
+static void
+CopyToTextBasedInit(CopyToState cstate)
+{
+}
and
+static void
+CopyToBinaryInit(CopyToState cstate)
+{
+}
Do we really need separate callbacks for the same behavior? I think we
can have a common init function say CopyToBuitinInit() that does
nothing. Or we can make the init callback optional.
The same is true for process-option callback.
---
List *convert_select; /* list of column names (can be NIL) */
+ const CopyToRoutine *to_routine; /* callback
routines for COPY TO */
} CopyFormatOptions;
I think CopyToStateData is a better place to have CopyToRoutine.
copy_data_dest_cb is also there.
---
- if (strcmp(fmt, "text") == 0)
- /* default format */ ;
- else if (strcmp(fmt, "csv") == 0)
- opts_out->csv_mode = true;
- else if (strcmp(fmt, "binary") == 0)
- opts_out->binary = true;
+
+ if (is_from)
+ {
+ char *fmt = defGetString(defel);
+
+ if (strcmp(fmt, "text") == 0)
+ /* default format */ ;
+ else if (strcmp(fmt, "csv") == 0)
+ {
+ opts_out->csv_mode = true;
+ }
+ else if (strcmp(fmt, "binary") == 0)
+ {
+ opts_out->binary = true;
+ }
else
- ereport(ERROR,
-
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format
\"%s\" not recognized", fmt\),
-
parser_errposition(pstate, defel->location)));
+ ProcessCopyOptionFormatTo(pstate,
opts_out, cstate, defel);
The 0002 patch replaces the options checks with
ProcessCopyOptionFormatFrom(). However, both
ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would
set format-related options such as opts_out->csv_mode etc, which seems
not elegant. IIUC the reason why we process only the "format" option
first is to set the callback functions and call the init callback. So
I think we don't necessarily need to do both setting callbacks and
setting format-related options together. Probably we can do only the
callback stuff first and then set format-related options in the
original place we used to do?
---
+static void
+CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf)
+{
+ int16 format = 0;
+ int natts = list_length(cstate->attnumlist);
+ int i;
+
+ pq_sendbyte(buf, format); /* overall format */
+ pq_sendint16(buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(buf, format); /* per-column formats */
+}
This function and CopyToBinaryFillCopyOutResponse() fill three things:
overall format, the number of columns, and per-column formats. While
this approach is flexible, extensions will have to understand the
format of CopyOutResponse message. An alternative is to have one or
more callbacks that return these three things.
---
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
sizeof(Fmgr\Info));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
Is preparing the out functions an extension's responsibility? I
thought the core could prepare them based on the overall format
specified by extensions, as long as the overall format matches the
actual data format to send. What do you think?
---
+ /*
+ * Called when COPY TO via the PostgreSQL protocol is
started. This must
+ * fill buf as a valid CopyOutResponse message:
+ *
+ */
+ /*--
+ * +--------+--------+--------+--------+--------+ +--------+--------+
+ * | Format | N attributes | Attr1's format |...| AttrN's format |
+ * +--------+--------+--------+--------+--------+ +--------+--------+
+ * 0: text 0: text 0: text
+ * 1: binary 1: binary 1: binary
+ */
I think this kind of diagram could be missed from being updated when
we update the CopyOutResponse format. It's better to refer to the
documentation instead.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-30 05:45 ` Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-30 05:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900,
Masahiko Sawada <[email protected]> wrote:
> ---
> + if (!format_specified)
> + /* Set the default format. */
> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
> +
>
> I think we can pass "text" in this case instead of NULL. That way,
> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
Yes, we can do it. But it needs a DefElem allocation. Is it
acceptable?
> We need curly brackets for this "if branch" as follows:
>
> if (!format_specifed)
> {
> /* Set the default format. */
> ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
> }
Oh, sorry. I assumed that pgindent adjusts the style too.
> ---
> + /* Process not built-in options. */
> + foreach(option, unknown_options)
> + {
> + DefElem *defel = lfirst_node(DefElem, option);
> + bool processed = false;
> +
> + if (!is_from)
> + processed =
> opts_out->to_routine->CopyToProcessOption(cstate, defel);
> + if (!processed)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("option \"%s\" not recognized",
> + defel->defname),
> + parser_errposition(pstate,
> defel->location)));
> + }
> + list_free(unknown_options);
>
> I think we can check the duplicated options in the core as we discussed.
Oh, sorry. I missed the part. I'll implement it.
> ---
> +static void
> +CopyToTextBasedInit(CopyToState cstate)
> +{
> +}
>
> and
>
> +static void
> +CopyToBinaryInit(CopyToState cstate)
> +{
> +}
>
> Do we really need separate callbacks for the same behavior? I think we
> can have a common init function say CopyToBuitinInit() that does
> nothing. Or we can make the init callback optional.
>
> The same is true for process-option callback.
OK. I'll make them optional.
> ---
> List *convert_select; /* list of column names (can be NIL) */
> + const CopyToRoutine *to_routine; /* callback
> routines for COPY TO */
> } CopyFormatOptions;
>
> I think CopyToStateData is a better place to have CopyToRoutine.
> copy_data_dest_cb is also there.
We can do it but ProcessCopyOptions() accepts NULL
CopyToState for file_fdw. Can we create an empty
CopyToStateData internally like we did for opts_out in
ProcessCopyOptions()? (But it requires exporting
CopyToStateData. We'll export it in a later patch but it's
not yet in 0001.)
> The 0002 patch replaces the options checks with
> ProcessCopyOptionFormatFrom(). However, both
> ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would
> set format-related options such as opts_out->csv_mode etc, which seems
> not elegant. IIUC the reason why we process only the "format" option
> first is to set the callback functions and call the init callback. So
> I think we don't necessarily need to do both setting callbacks and
> setting format-related options together. Probably we can do only the
> callback stuff first and then set format-related options in the
> original place we used to do?
If we do it, we need to write the (strcmp(format, "csv") ==
0) condition in copyto.c and copy.c. I wanted to avoid it. I
think that the duplication (setting opts_out->csv_mode in
copyto.c and copyfrom.c) is not a problem. But it's not a
strong opinion. If (strcmp(format, "csv") == 0) duplication
is better than opts_out->csv_mode = true duplication, I'll
do it.
BTW, if we want to make the CSV format implementation more
modularized, we will remove opts_out->csv_mode, move CSV
related options to CopyToCSVProcessOption() and keep CSV
related options in its opaque space. For example,
opts_out->force_quote exists in COPY TO opaque space but
doesn't exist in COPY FROM opaque space because it's not
used in COPY FROM.
> +static void
> +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf)
> +{
> + int16 format = 0;
> + int natts = list_length(cstate->attnumlist);
> + int i;
> +
> + pq_sendbyte(buf, format); /* overall format */
> + pq_sendint16(buf, natts);
> + for (i = 0; i < natts; i++)
> + pq_sendint16(buf, format); /* per-column formats */
> +}
>
> This function and CopyToBinaryFillCopyOutResponse() fill three things:
> overall format, the number of columns, and per-column formats. While
> this approach is flexible, extensions will have to understand the
> format of CopyOutResponse message. An alternative is to have one or
> more callbacks that return these three things.
Yes, we can choose the approach. I don't have a strong
opinion on which approach to choose.
> + /* Get info about the columns we need to process. */
> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
> sizeof(Fmgr\Info));
> + foreach(cur, cstate->attnumlist)
> + {
> + int attnum = lfirst_int(cur);
> + Oid out_func_oid;
> + bool isvarlena;
> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
> +
> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
> + }
>
> Is preparing the out functions an extension's responsibility? I
> thought the core could prepare them based on the overall format
> specified by extensions, as long as the overall format matches the
> actual data format to send. What do you think?
Hmm. I want to keep the preparation as an extension's
responsibility. Because it's not needed for all formats. For
example, Apache Arrow FORMAT doesn't need it. And JSON
FORMAT doesn't need it too because it use
composite_to_json().
> + /*
> + * Called when COPY TO via the PostgreSQL protocol is
> started. This must
> + * fill buf as a valid CopyOutResponse message:
> + *
> + */
> + /*--
> + * +--------+--------+--------+--------+--------+ +--------+--------+
> + * | Format | N attributes | Attr1's format |...| AttrN's format |
> + * +--------+--------+--------+--------+--------+ +--------+--------+
> + * 0: text 0: text 0: text
> + * 1: binary 1: binary 1: binary
> + */
>
> I think this kind of diagram could be missed from being updated when
> we update the CopyOutResponse format. It's better to refer to the
> documentation instead.
It makes sense. I couldn't find the documentation when I
wrote it but I found it now...:
https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY
Is there recommended comment style to refer a documentation?
"See doc/src/sgml/protocol.sgml for the CopyOutResponse
message details" is OK?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 07:20 ` Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-30 07:20 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jan 30, 2024 at 02:45:31PM +0900, Sutou Kouhei wrote:
> In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
>> ---
>> + if (!format_specified)
>> + /* Set the default format. */
>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
>> +
>>
>> I think we can pass "text" in this case instead of NULL. That way,
>> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
>
> Yes, we can do it. But it needs a DefElem allocation. Is it
> acceptable?
I don't think that there is a need for a DelElem at all here? While I
am OK with the choice of calling CopyToInit() in the
ProcessCopyOption*() routines that exist to keep the set of callbacks
local to copyto.c and copyfrom.c, I think that this should not bother
about setting opts_out->csv_mode or opts_out->csv_mode but just set
the opts_out->{to,from}_routine callbacks.
>> +static void
>> +CopyToTextBasedInit(CopyToState cstate)
>> +{
>> +}
>>
>> and
>>
>> +static void
>> +CopyToBinaryInit(CopyToState cstate)
>> +{
>> +}
>>
>> Do we really need separate callbacks for the same behavior? I think we
>> can have a common init function say CopyToBuitinInit() that does
>> nothing. Or we can make the init callback optional.
Keeping empty options does not strike as a bad idea, because this
forces extension developers to think about this code path rather than
just ignore it. Now, all the Init() callbacks are empty for the
in-core callbacks, so I think that we should just remove it entirely
for now. Let's keep the core patch a maximum simple. It is always
possible to build on top of it depending on what people need. It's
been mentioned that JSON would want that, but this also proves that we
just don't care about that for all the in-core callbacks, as well. I
would choose a minimalistic design for now.
>> + /* Get info about the columns we need to process. */
>> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
>> sizeof(Fmgr\Info));
>> + foreach(cur, cstate->attnumlist)
>> + {
>> + int attnum = lfirst_int(cur);
>> + Oid out_func_oid;
>> + bool isvarlena;
>> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
>> +
>> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
>> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>> + }
>>
>> Is preparing the out functions an extension's responsibility? I
>> thought the core could prepare them based on the overall format
>> specified by extensions, as long as the overall format matches the
>> actual data format to send. What do you think?
>
> Hmm. I want to keep the preparation as an extension's
> responsibility. Because it's not needed for all formats. For
> example, Apache Arrow FORMAT doesn't need it. And JSON
> FORMAT doesn't need it too because it use
> composite_to_json().
I agree that it could be really useful for extensions to be able to
force that. We already know that for the in-core formats we've cared
about being able to enforce the way data is handled in input and
output.
> It makes sense. I couldn't find the documentation when I
> wrote it but I found it now...:
> https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY
>
> Is there recommended comment style to refer a documentation?
> "See doc/src/sgml/protocol.sgml for the CopyOutResponse
> message details" is OK?
There are a couple of places in the C code where we refer to SGML docs
when it comes to specific details, so using a method like that here to
avoid a duplication with the docs sounds sensible for me.
I would be really tempted to put my hands on this patch to put into
shape a minimal set of changes because I'm caring quite a lot about
the performance gains reported with the removal of the "if" checks in
the per-row callbacks, and that's one goal of this thread quite
independent on the extensibility. Sutou-san, would you be OK with
that?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-30 08:15 ` Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-30 08:15 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 16:20:54 +0900,
Michael Paquier <[email protected]> wrote:
>>> + if (!format_specified)
>>> + /* Set the default format. */
>>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
>>> +
>>>
>>> I think we can pass "text" in this case instead of NULL. That way,
>>> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
>>
>> Yes, we can do it. But it needs a DefElem allocation. Is it
>> acceptable?
>
> I don't think that there is a need for a DelElem at all here?
We use defel->location for an error message. (We don't need
to set location for the default "text" DefElem.)
> While I
> am OK with the choice of calling CopyToInit() in the
> ProcessCopyOption*() routines that exist to keep the set of callbacks
> local to copyto.c and copyfrom.c, I think that this should not bother
> about setting opts_out->csv_mode or opts_out->csv_mode but just set
> the opts_out->{to,from}_routine callbacks.
OK. I'll keep opts_out->{csv_mode,binary} in copy.c.
> Now, all the Init() callbacks are empty for the
> in-core callbacks, so I think that we should just remove it entirely
> for now. Let's keep the core patch a maximum simple. It is always
> possible to build on top of it depending on what people need. It's
> been mentioned that JSON would want that, but this also proves that we
> just don't care about that for all the in-core callbacks, as well. I
> would choose a minimalistic design for now.
OK. Let's remove Init() callbacks from the first patch set.
> I would be really tempted to put my hands on this patch to put into
> shape a minimal set of changes because I'm caring quite a lot about
> the performance gains reported with the removal of the "if" checks in
> the per-row callbacks, and that's one goal of this thread quite
> independent on the extensibility. Sutou-san, would you be OK with
> that?
Yes, sure.
(We want to focus on the performance gains in the first
patch set and then focus on extensibility again, right?)
For the purpose, I think that the v7 patch set is more
suitable than the v9 patch set. The v7 patch set doesn't
include Init() callbacks, custom options validation support
or extra Copy{In,Out}Response support. But the v7 patch set
misses the removal of the "if" checks in
NextCopyFromRawFields() that exists in the v9 patch set. I'm
not sure how much performance will improve by this but it
may be worth a try.
Can I prepare the v10 patch set as "the v7 patch set" + "the
removal of the "if" checks in NextCopyFromRawFields()"?
(+ reverting opts_out->{csv_mode,binary} changes in
ProcessCopyOptions().)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 08:37 ` Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2024-01-30 08:37 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jan 30, 2024 at 05:15:11PM +0900, Sutou Kouhei wrote:
> We use defel->location for an error message. (We don't need
> to set location for the default "text" DefElem.)
Yeah, but you should not need to have this error in the paths that set
the callback routines in opts_out if the same validation happens a few
lines before, in copy.c.
> Yes, sure.
> (We want to focus on the performance gains in the first
> patch set and then focus on extensibility again, right?)
Yep, exactly, the numbers are too good to just ignore. I don't want
to hijack the thread, but I am really worried about the complexities
this thread is getting into because we are trying to shape the
callbacks in the most generic way possible based on *two* use cases.
This is going to be a never-ending discussion. I'd rather get some
simple basics, and then we can discuss if tweaking the callbacks is
really necessary or not. Even after introducing the pg_proc lookups
to get custom callbacks.
> For the purpose, I think that the v7 patch set is more
> suitable than the v9 patch set. The v7 patch set doesn't
> include Init() callbacks, custom options validation support
> or extra Copy{In,Out}Response support. But the v7 patch set
> misses the removal of the "if" checks in
> NextCopyFromRawFields() that exists in the v9 patch set. I'm
> not sure how much performance will improve by this but it
> may be worth a try.
Yeah.. The custom options don't seem like an absolute strong
requirement for the first shot with the callbacks or even the
possibility to retrieve the callbacks from a function call. I mean,
you could provide some control with SET commands and a few GUCs, at
least, even if that would be strange. Manipulations with a list of
DefElems is the intuitive way to have custom options at query level,
but we also have to guess the set of callbacks from this list of
DefElems coming from the query. You see my point, I am not sure
if it would be the best thing to process twice the options, especially
when it comes to decide if a DefElem should be valid or not depending
on the callbacks used. Or we could use a kind of "special" DefElem
where we could store a set of key:value fed to a callback :)
> Can I prepare the v10 patch set as "the v7 patch set" + "the
> removal of the "if" checks in NextCopyFromRawFields()"?
> (+ reverting opts_out->{csv_mode,binary} changes in
> ProcessCopyOptions().)
Yep, if I got it that would make sense to me. If you can do that,
that would help quite a bit. :)
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-31 05:11 ` Sutou Kouhei <[email protected]>
2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-31 05:11 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 17:37:35 +0900,
Michael Paquier <[email protected]> wrote:
>> We use defel->location for an error message. (We don't need
>> to set location for the default "text" DefElem.)
>
> Yeah, but you should not need to have this error in the paths that set
> the callback routines in opts_out if the same validation happens a few
> lines before, in copy.c.
Ah, yes. defel->location is used in later patches. For
example, it's used when a COPY handler for the specified
FORMAT isn't found.
> I am really worried about the complexities
> this thread is getting into because we are trying to shape the
> callbacks in the most generic way possible based on *two* use cases.
> This is going to be a never-ending discussion. I'd rather get some
> simple basics, and then we can discuss if tweaking the callbacks is
> really necessary or not. Even after introducing the pg_proc lookups
> to get custom callbacks.
I understand your concern. Let's introduce minimal callbacks
as the first step. I think that we completed our design
discussion for this feature. We can choose minimal callbacks
based on the discussion.
> The custom options don't seem like an absolute strong
> requirement for the first shot with the callbacks or even the
> possibility to retrieve the callbacks from a function call. I mean,
> you could provide some control with SET commands and a few GUCs, at
> least, even if that would be strange. Manipulations with a list of
> DefElems is the intuitive way to have custom options at query level,
> but we also have to guess the set of callbacks from this list of
> DefElems coming from the query. You see my point, I am not sure
> if it would be the best thing to process twice the options, especially
> when it comes to decide if a DefElem should be valid or not depending
> on the callbacks used. Or we could use a kind of "special" DefElem
> where we could store a set of key:value fed to a callback :)
Interesting. Let's remove custom options support from the
initial minimal callbacks.
>> Can I prepare the v10 patch set as "the v7 patch set" + "the
>> removal of the "if" checks in NextCopyFromRawFields()"?
>> (+ reverting opts_out->{csv_mode,binary} changes in
>> ProcessCopyOptions().)
>
> Yep, if I got it that would make sense to me. If you can do that,
> that would help quite a bit. :)
I've prepared the v10 patch set. Could you try this?
Changes since the v7 patch set:
0001:
* Remove CopyToProcessOption() callback
* Remove CopyToGetFormat() callback
* Revert passing CopyToState to ProcessCopyOptions()
* Revert moving "opts_out->{csv_mode,binary} = true" to
ProcessCopyOptionFormatTo()
* Change to receive "const char *format" instead "DefElem *defel"
by ProcessCopyOptionFormatTo()
0002:
* Remove CopyFromProcessOption() callback
* Remove CopyFromGetFormat() callback
* Change to receive "const char *format" instead "DefElem
*defel" by ProcessCopyOptionFormatFrom()
* Remove "if (cstate->opts.csv_mode)" branches from
NextCopyFromRawFields()
FYI: Here are Copy{From,To}Routine in the v10 patch set. I
think that only Copy{From,To}OneRow are minimal callbacks
for the performance gain. But can we keep Copy{From,To}Start
and Copy{From,To}End for consistency? We can remove a few
{csv_mode,binary} conditions by Copy{From,To}{Start,End}. It
doesn't depend on the number of COPY target tuples. So they
will not affect performance.
/* Routines for a COPY FROM format implementation. */
typedef struct CopyFromRoutine
{
/*
* Called when COPY FROM is started. This will initialize something and
* receive a header.
*/
void (*CopyFromStart) (CopyFromState cstate, TupleDesc tupDesc);
/* Copy one row. It returns false if no more tuples. */
bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
/* Called when COPY FROM is ended. This will finalize something. */
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
/* Routines for a COPY TO format implementation. */
typedef struct CopyToRoutine
{
/* Called when COPY TO is started. This will send a header. */
void (*CopyToStart) (CopyToState cstate, TupleDesc tupDesc);
/* Copy one row for COPY TO. */
void (*CopyToOneRow) (CopyToState cstate, TupleTableSlot *slot);
/* Called when COPY TO is ended. This will send a trailer. */
void (*CopyToEnd) (CopyToState cstate);
} CopyToRoutine;
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-31 05:39 ` Michael Paquier <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Michael Paquier @ 2024-01-31 05:39 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 31, 2024 at 02:11:22PM +0900, Sutou Kouhei wrote:
> Ah, yes. defel->location is used in later patches. For
> example, it's used when a COPY handler for the specified
> FORMAT isn't found.
I see.
> I've prepared the v10 patch set. Could you try this?
Thanks, I'm looking into that now.
> FYI: Here are Copy{From,To}Routine in the v10 patch set. I
> think that only Copy{From,To}OneRow are minimal callbacks
> for the performance gain. But can we keep Copy{From,To}Start
> and Copy{From,To}End for consistency? We can remove a few
> {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It
> doesn't depend on the number of COPY target tuples. So they
> will not affect performance.
I think I'm OK to keep the start/end callbacks. This makes the code
more consistent as a whole, as well.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-24 14:20 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2024-01-24 14:20 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 14:49:36 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
> I've implemented custom COPY format feature based on the
> current design discussion. See the attached patches for
> details.
I forgot to mention one note. Documentation isn't included
in these patches. I'll write it after all (or some) patches
are merged. Is it OK?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
@ 2025-03-17 20:50 Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-17 20:50 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Mar 4, 2025 at 4:06 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoAwOP7p6LgmkPGqPuJ5KbJPPQsSZsFzwCDguwzr9F677Q@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 3 Mar 2025 11:06:39 -0800,
> Masahiko Sawada <[email protected]> wrote:
>
> > I agree with the fix and the patch looks good to me. I've updated the
> > commit message and am going to push, barring any objections.
>
> Thanks!
>
> I've rebased the patch set. Here is a summary again:
Thank you for updating the patches. Here are some review comments on
the 0001 patch:
+ if (strcmp(format, "text") == 0)
+ {
+ /* "csv_mode == false && binary == false" means "text" */
+ return;
+ }
+ else if (strcmp(format, "csv") == 0)
+ {
+ opts_out->csv_mode = true;
+ return;
+ }
+ else if (strcmp(format, "binary") == 0)
+ {
+ opts_out->binary = true;
+ return;
+ }
+
+ /* custom format */
+ if (!is_from)
+ {
+ funcargtypes[0] = INTERNALOID;
+ handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
+ funcargtypes, true);
+ }
+ if (!OidIsValid(handlerOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", format),
+ parser_errposition(pstate, defel->location)));
I think that built-in formats also need to have their handler
functions. This seems to be a conventional way for customizable
features such as tablesample and access methods, and we can simplify
this function.
---
I think we need to update the documentation to describe how users can
define the handler functions and what each callback function is
responsible for.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-19 02:56 ` Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-03-19 02:56 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Mar 2025 13:50:03 -0700,
Masahiko Sawada <[email protected]> wrote:
> I think that built-in formats also need to have their handler
> functions. This seems to be a conventional way for customizable
> features such as tablesample and access methods, and we can simplify
> this function.
OK. 0008 in the attached v37 patch set does it.
> I think we need to update the documentation to describe how users can
> define the handler functions and what each callback function is
> responsible for.
I agree with it but we haven't finalized public APIs yet. Can
we defer it after we finalize public APIs? (Proposed public
APIs exist in 0003, 0006 and 0007.)
And could someone help (take over if possible) writing a
document for this feature? I'm not good at writing a
document in English... 0009 in the attached v37 patch set
has a draft of it. It's based on existing documents in
doc/src/sgml/ and *.h.
0001-0007 aren't changed from v36 patch set.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-20 00:49 ` David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-03-20 00:49 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote:
> And could someone help (take over if possible) writing a
> document for this feature? I'm not good at writing a
> document in English... 0009 in the attached v37 patch set
> has a draft of it. It's based on existing documents in
> doc/src/sgml/ and *.h.
>
>
I haven't touched the innards of the structs aside from changing
programlisting to synopsis. And redoing the two section opening paragraphs
to better integrate with the content in the chapter opening.
The rest I kinda went to town on...
David J.
diff --git a/doc/src/sgml/copy-handler.sgml b/doc/src/sgml/copy-handler.sgml
index f602debae6..9d2897a104 100644
--- a/doc/src/sgml/copy-handler.sgml
+++ b/doc/src/sgml/copy-handler.sgml
@@ -10,56 +10,72 @@
<para>
<productname>PostgreSQL</productname> supports
custom <link linkend="sql-copy"><literal>COPY</literal></link>
- handlers. The <literal>COPY</literal> handlers can use different copy
format
- instead of built-in <literal>text</literal>, <literal>csv</literal>
- and <literal>binary</literal>.
+ handlers; adding additional <replaceable>format_name</replaceable>
options
+ to the <literal>FORMAT</literal> clause.
</para>
<para>
- At the SQL level, a table sampling method is represented by a single SQL
- function, typically implemented in C, having the signature
-<programlisting>
-format_name(internal) RETURNS copy_handler
-</programlisting>
- The name of the function is the same name appearing in
- the <literal>FORMAT</literal> option. The <type>internal</type> argument
is
- a dummy that simply serves to prevent this function from being called
- directly from an SQL command. The real argument is <literal>bool
- is_from</literal>. If the handler is used by <literal>COPY
FROM</literal>,
- it's <literal>true</literal>. If the handler is used by <literal>COPY
- FROM</literal>, it's <literal>false</literal>.
+ At the SQL level, a copy handler method is represented by a single SQL
+ function (see <xref linkend="sql-createfunction"/>), typically
implemented in
+ C, having the signature
+<synopsis>
+<replaceable>format_name</replaceable>(internal) RETURNS
<literal>copy_handler</literal>
+</synopsis>
+ The function's name is then accepted as a valid
<replaceable>format_name</replaceable>.
+ The return pseudo-type <literal>copy_handler</literal> informs the
system that
+ this function needs to be registered as a copy handler.
+ The <type>internal</type> argument is a dummy that prevents
+ this function from being called directly from an SQL command. As the
+ handler implementation must be server-lifetime immutable; this SQL
function's
+ volatility should be marked immutable. The
<literal>link_symbol</literal>
+ for this function is the name of the implementation function, described
next.
</para>
<para>
- The function must return <type>CopyFromRoutine *</type> when
- the <literal>is_from</literal> argument is <literal>true</literal>.
- The function must return <type>CopyToRoutine *</type> when
- the <literal>is_from</literal> argument is <literal>false</literal>.
+ The implementation function signature expected for the function named
+ in the <literal>link_symbol</literal> is:
+<synopsis>
+Datum
+<replaceable>copy_format_handler</replaceable>(PG_FUNCTION_ARGS)
+</synopsis>
+ The convention for the name is to replace the word
+ <replaceable>format</replaceable> in the placeholder above with the
value given
+ to <replaceable>format_name</replaceable> in the SQL function.
+ The first argument is a <type>boolean</type> that indicates whether the
handler
+ must provide a pointer to its implementation for <literal>COPY
FROM</literal>
+ (a <type>CopyFromRoutine *</type>). If <literal>false</literal>, the
handler
+ must provide a pointer to its implementation of <literal>COPY
TO</literal>
+ (a <type>CopyToRoutine *</type>). These structs are declared in
+ <filename>src/include/commands/copyapi.h</filename>.
</para>
<para>
- The <type>CopyFromRoutine</type> and <type>CopyToRoutine</type> struct
types
- are declared in <filename>src/include/commands/copyapi.h</filename>,
- which see for additional details.
+ The structs hold pointers to implementation functions for
+ initializing, starting, processing rows, and ending a copy operation.
+ The specific structures vary a bit between <literal>COPY FROM</literal>
and
+ <literal>COPY TO</literal> so the next two sections describes each
+ in detail.
</para>
<sect1 id="copy-handler-from">
<title>Copy From Handler</title>
<para>
- The <literal>COPY</literal> handler function for <literal>COPY
- FROM</literal> returns a <type>CopyFromRoutine</type> struct containing
- pointers to the functions described below. All functions are required.
+ The opening to this chapter describes how the executor will call the
+ main handler function with, in this case,
+ a <type>boolean</type> <literal>true</literal>, and expect to receive a
+ <type>CopyFromRoutine *</type> <type>Datum</type>. This section
describes
+ the components of the <type>CopyFromRoutine</type> struct.
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyFromInFunc(CopyFromState cstate,
Oid atttypid,
FmgrInfo *finfo,
Oid *typioparam);
-</programlisting>
+</synopsis>
This sets input function information for the
given <literal>atttypid</literal> attribute. This function is called
once
@@ -110,11 +126,11 @@ CopyFromInFunc(CopyFromState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyFromStart(CopyFromState cstate,
TupleDesc tupDesc);
-</programlisting>
+</synopsis>
This starts a <literal>COPY FROM</literal>. This function is called
once at
the beginning of <literal>COPY FROM</literal>.
@@ -144,13 +160,13 @@ CopyFromStart(CopyFromState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
bool
CopyFromOneRow(CopyFromState cstate,
ExprContext *econtext,
Datum *values,
bool *nulls);
-</programlisting>
+</synopsis>
This reads one row from the source and fill <literal>values</literal>
and <literal>nulls</literal>. If there is one or more tuples to be read,
@@ -202,10 +218,10 @@ CopyFromOneRow(CopyFromState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyFromEnd(CopyFromState cstate);
-</programlisting>
+</synopsis>
This ends a <literal>COPY FROM</literal>. This function is called once
at
the end of <literal>COPY FROM</literal>.
@@ -232,18 +248,20 @@ CopyFromEnd(CopyFromState cstate);
<title>Copy To Handler</title>
<para>
- The <literal>COPY</literal> handler function for <literal>COPY
- TO</literal> returns a <type>CopyToRoutine</type> struct containing
- pointers to the functions described below. All functions are required.
+ The opening to this chapter describes how the executor will call the
+ main handler function with, in this case,
+ a <type>boolean</type> <literal>false</literal>, and expect to receive a
+ <type>CopyInRoutine *</type> <type>Datum</type>. This section describes
+ the components of the <type>CopyInRoutine</type> struct.
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyToOutFunc(CopyToState cstate,
Oid atttypid,
FmgrInfo *finfo);
-</programlisting>
+</synopsis>
This sets output function information for the
given <literal>atttypid</literal> attribute. This function is called
once
@@ -284,11 +302,11 @@ CopyToOutFunc(CopyToState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyToStart(CopyToState cstate,
TupleDesc tupDesc);
-</programlisting>
+</synopsis>
This starts a <literal>COPY TO</literal>. This function is called once
at
the beginning of <literal>COPY TO</literal>.
@@ -316,11 +334,11 @@ CopyToStart(CopyToState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
bool
CopyToOneRow(CopyToState cstate,
TupleTableSlot *slot);
-</programlisting>
+</synopsis>
This writes one row stored in <literal>slot</literal> to the
destination.
@@ -347,10 +365,10 @@ CopyToOneRow(CopyToState cstate,
</para>
<para>
-<programlisting>
+<synopsis>
void
CopyToEnd(CopyToState cstate);
-</programlisting>
+</synopsis>
This ends a <literal>COPY TO</literal>. This function is called once at
the end of <literal>COPY TO</literal>.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-03-20 01:24 ` Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-25 00:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-03-20 01:24 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700,
"David G. Johnston" <[email protected]> wrote:
>> And could someone help (take over if possible) writing a
>> document for this feature? I'm not good at writing a
>> document in English... 0009 in the attached v37 patch set
>> has a draft of it. It's based on existing documents in
>> doc/src/sgml/ and *.h.
>>
>>
> I haven't touched the innards of the structs aside from changing
> programlisting to synopsis. And redoing the two section opening paragraphs
> to better integrate with the content in the chapter opening.
>
> The rest I kinda went to town on...
Thanks!!! It's very helpful!!!
I've applied your patch. 0009 is only changed.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-23 09:01 ` Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-23 09:01 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Mar 19, 2025 at 6:25 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700,
> "David G. Johnston" <[email protected]> wrote:
>
> >> And could someone help (take over if possible) writing a
> >> document for this feature? I'm not good at writing a
> >> document in English... 0009 in the attached v37 patch set
> >> has a draft of it. It's based on existing documents in
> >> doc/src/sgml/ and *.h.
> >>
> >>
> > I haven't touched the innards of the structs aside from changing
> > programlisting to synopsis. And redoing the two section opening paragraphs
> > to better integrate with the content in the chapter opening.
> >
> > The rest I kinda went to town on...
>
> Thanks!!! It's very helpful!!!
>
> I've applied your patch. 0009 is only changed.
Thank you for updating the patches. I've reviewed the main part of
supporting the custom COPY format. Here are some random comments:
---
+/*
+ * Process the "format" option.
+ *
+ * This function checks whether the option value is a built-in format such as
+ * "text" and "csv" or not. If the option value isn't a built-in format, this
+ * function finds a COPY format handler that returns a CopyToRoutine (for
+ * is_from == false) or CopyFromRountine (for is_from == true). If no COPY
+ * format handler is found, this function reports an error.
+ */
I think this comment needs to be updated as the part "If the option
value isn't ..." is no longer true.
I think we don't necessarily need to create a separate function
ProcessCopyOptionFormat for processing the format option.
We need more regression tests for handling the given format name. For example,
- more various input patterns.
- a function with the specified format name exists but it returns an
unexpected Node.
- looking for a handler function in a different namespace.
etc.
---
I think that we should accept qualified names too as the format name
like tablesample does. That way, different extensions implementing the
same format can be used.
---
+ if (routine == NULL || !IsA(routine, CopyFromRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+
ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%u did not return "
+ "CopyFromRoutine struct",
+ opts->handler)));
It's not conventional to put a new line between 'ereport(' and 'ERROR'
(similarly between 'errcode(' and 'ERRCODE_...'. Also, we don't need
to split the error message into multiple lines as it's not long.
---
+ if (routine == NULL || !IsA(routine, CopyToRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+
ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%u did not return "
+ "CopyToRoutine struct",
+ opts->handler)));
Same as the above comment.
---
+ descr => 'pseudo-type for the result of a copy to/from method function',
s/method function/format function/
---
+ Oid handler; /* handler
function for custom format routine */
'handler' is used also for built-in formats.
---
+static void
+CopyFromInFunc(CopyFromState cstate, Oid atttypid,
+ FmgrInfo *finfo, Oid *typioparam)
+{
+ ereport(NOTICE, (errmsg("CopyFromInFunc: atttypid=%d", atttypid)));
+}
OIDs could be changed across major versions even for built-in types. I
think it's better to avoid using it for tests.
---
+static void
+CopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u",
slot->tts_nvalid)));
+}
Similar to the above comment, the field name 'tts_nvalid' might also
be changed in the future, let's use another name.
---
+static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
+ .type = T_CopyFromRoutine,
+ .CopyFromInFunc = CopyFromInFunc,
+ .CopyFromStart = CopyFromStart,
+ .CopyFromOneRow = CopyFromOneRow,
+ .CopyFromEnd = CopyFromEnd,
+};
I'd suggest not using the same function names as the fields.
---
+/*
+ * Export CopySendEndOfRow() for extensions. We want to keep
+ * CopySendEndOfRow() as a static function for
+ * optimization. CopySendEndOfRow() calls in this file may be optimized by a
+ * compiler.
+ */
+void
+CopyToStateFlush(CopyToState cstate)
+{
+ CopySendEndOfRow(cstate);
+}
Is there any reason to use a different name for public functions?
---
+/*
+ * Export CopyGetData() for extensions. We want to keep CopyGetData() as a
+ * static function for optimization. CopyGetData() calls in this file may be
+ * optimized by a compiler.
+ */
+int
+CopyFromStateGetData(CopyFromState cstate, void *dest, int minread,
int maxread)
+{
+ return CopyGetData(cstate, dest, minread, maxread);
+}
+
The same as the above comment.
---
+ /* For custom format implementation */
+ void *opaque; /* private space */
How about renaming 'private'?
---
I've not reviewed the documentation patch yet but I think the patch
seems to miss the updates to the description of the FORMAT option in
the COPY command section.
---
I think we can reorganize the patch set as follows:
1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and
COPY_DEST_XXX accordingly.
2. Support custom format for both COPY TO and COPY FROM.
3. Expose necessary helper functions such as CopySendEndOfRow().
4. Add CopyFromSkipErrorRow().
5. Documentation.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-27 03:28 ` Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-31 17:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-04-06 16:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 4 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-03-27 03:28 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoAfWrjpTDJ0garVUoXY0WC3Ud4Cu51q+ccWiotm1uo_2A@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Sun, 23 Mar 2025 02:01:59 -0700,
Masahiko Sawada <[email protected]> wrote:
> ---
> +/*
> + * Process the "format" option.
> + *
> + * This function checks whether the option value is a built-in format such as
> + * "text" and "csv" or not. If the option value isn't a built-in format, this
> + * function finds a COPY format handler that returns a CopyToRoutine (for
> + * is_from == false) or CopyFromRountine (for is_from == true). If no COPY
> + * format handler is found, this function reports an error.
> + */
>
> I think this comment needs to be updated as the part "If the option
> value isn't ..." is no longer true.
>
> I think we don't necessarily need to create a separate function
> ProcessCopyOptionFormat for processing the format option.
Hmm. I think that this separated function will increase
readability by reducing indentation. But I've removed the
separation as you suggested. So the comment is also removed
entirely.
0002 includes this.
> We need more regression tests for handling the given format name. For example,
>
> - more various input patterns.
> - a function with the specified format name exists but it returns an
> unexpected Node.
> - looking for a handler function in a different namespace.
> etc.
I've added the following tests:
* Wrong input type handler without namespace
* Wrong input type handler with namespace
* Wrong return type handler without namespace
* Wrong return type handler with namespace
* Wrong return value (Copy*Routine isn't returned) handler without namespace
* Wrong return value (Copy*Routine isn't returned) handler with namespace
* Nonexistent handler
* Invalid qualified name
* Valid handler without namespace and without search_path
* Valid handler without namespace and with search_path
* Valid handler with namespace
0002 also includes this.
> I think that we should accept qualified names too as the format name
> like tablesample does. That way, different extensions implementing the
> same format can be used.
Implemented. It's implemented after parsing SQL. Is it OK?
(It seems that tablesample does it in parsing SQL.)
Because "WITH (FORMAT XXX)" is processed as a generic option
in gram.y. All generic options are processed as strings. So
I keep this.
Syntax is "COPY ... WITH (FORMAT 'NAMESPACE.HANDLER_NAME')"
not "COPY ... WITH (FORMAT 'NAMESPACE'.'HANDLER_NAME')"
because of this choice.
0002 also includes this.
> ---
> + if (routine == NULL || !IsA(routine, CopyFromRoutine))
> + ereport(
> + ERROR,
> + (errcode(
> +
> ERRCODE_INVALID_PARAMETER_VALUE),
> + errmsg("COPY handler function "
> + "%u did not return "
> + "CopyFromRoutine struct",
> + opts->handler)));
>
> It's not conventional to put a new line between 'ereport(' and 'ERROR'
> (similarly between 'errcode(' and 'ERRCODE_...'. Also, we don't need
> to split the error message into multiple lines as it's not long.
Oh, sorry. I can't remember why I used this... I think I
trusted pgindent...
> ---
> + descr => 'pseudo-type for the result of a copy to/from method function',
>
> s/method function/format function/
Good catch. I used "handler function" not "format function"
because we use "handler" in other places.
> ---
> + Oid handler; /* handler
> function for custom format routine */
>
> 'handler' is used also for built-in formats.
Updated in 0004.
> ---
> +static void
> +CopyFromInFunc(CopyFromState cstate, Oid atttypid,
> + FmgrInfo *finfo, Oid *typioparam)
> +{
> + ereport(NOTICE, (errmsg("CopyFromInFunc: atttypid=%d", atttypid)));
> +}
>
> OIDs could be changed across major versions even for built-in types. I
> think it's better to avoid using it for tests.
Oh, I didn't know it. I've changed to use type name instead
of OID. It'll be more stable than OID.
> ---
> +static void
> +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
> +{
> + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u",
> slot->tts_nvalid)));
> +}
>
> Similar to the above comment, the field name 'tts_nvalid' might also
> be changed in the future, let's use another name.
Hmm. If the field name is changed, we need to change this
code. So changing tests too isn't strange. Anyway, I used
more generic text.
> ---
> +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
> + .type = T_CopyFromRoutine,
> + .CopyFromInFunc = CopyFromInFunc,
> + .CopyFromStart = CopyFromStart,
> + .CopyFromOneRow = CopyFromOneRow,
> + .CopyFromEnd = CopyFromEnd,
> +};
>
> I'd suggest not using the same function names as the fields.
OK. I've added "Test" prefix.
> ---
> +/*
> + * Export CopySendEndOfRow() for extensions. We want to keep
> + * CopySendEndOfRow() as a static function for
> + * optimization. CopySendEndOfRow() calls in this file may be optimized by a
> + * compiler.
> + */
> +void
> +CopyToStateFlush(CopyToState cstate)
> +{
> + CopySendEndOfRow(cstate);
> +}
>
> Is there any reason to use a different name for public functions?
In this patch set, I use "CopyFrom"/"CopyTo" prefixes for
public APIs for custom COPY FORMAT handler extensions. It
will help understanding related APIs. Is it strange in
PostgreSQL?
> ---
> + /* For custom format implementation */
> + void *opaque; /* private space */
>
> How about renaming 'private'?
We should not use "private" because it's a keyword in
C++. If we use "private" here, we can't include this file
from C++ code.
> ---
> I've not reviewed the documentation patch yet but I think the patch
> seems to miss the updates to the description of the FORMAT option in
> the COPY command section.
I defer this for now. We can revisit the last documentation
patch after we finalize our API. (Or could someone help us?)
> I think we can reorganize the patch set as follows:
>
> 1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and
> COPY_DEST_XXX accordingly.
> 2. Support custom format for both COPY TO and COPY FROM.
> 3. Expose necessary helper functions such as CopySendEndOfRow().
> 4. Add CopyFromSkipErrorRow().
> 5. Documentation.
The attached v39 patch set uses the followings:
0001: Create copyto_internal.h and change COPY_XXX to
COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
(Same as 1. in your suggestion)
0002: Support custom format for both COPY TO and COPY FROM.
(Same as 2. in your suggestion)
0003: Expose necessary helper functions such as CopySendEndOfRow()
and add CopyFromSkipErrorRow().
(3. + 4. in your suggestion)
0004: Define handler functions for built-in formats.
(Not included in your suggestion)
0005: Documentation. (WIP)
(Same as 5. in your suggestion)
We can merge 0001 quickly, right?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-29 05:37 ` Masahiko Sawada <[email protected]>
2025-03-29 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
3 siblings, 2 replies; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-29 05:37 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote:
>
> > We need more regression tests for handling the given format name. For example,
> >
> > - more various input patterns.
> > - a function with the specified format name exists but it returns an
> > unexpected Node.
> > - looking for a handler function in a different namespace.
> > etc.
>
> I've added the following tests:
>
> * Wrong input type handler without namespace
> * Wrong input type handler with namespace
> * Wrong return type handler without namespace
> * Wrong return type handler with namespace
> * Wrong return value (Copy*Routine isn't returned) handler without namespace
> * Wrong return value (Copy*Routine isn't returned) handler with namespace
> * Nonexistent handler
> * Invalid qualified name
> * Valid handler without namespace and without search_path
> * Valid handler without namespace and with search_path
> * Valid handler with namespace
Probably we can merge these newly added four files into one .sql file?
Also we need to add some comments to describe what these queries test.
For example, it's not clear to me at a glance what queries in
no-schema.sql are going to test as there is no comment there.
>
> 0002 also includes this.
>
> > I think that we should accept qualified names too as the format name
> > like tablesample does. That way, different extensions implementing the
> > same format can be used.
>
> Implemented. It's implemented after parsing SQL. Is it OK?
> (It seems that tablesample does it in parsing SQL.)
I think it's okay.
One problem in the following chunk I can see is:
+ qualified_format = stringToQualifiedNameList(format, NULL);
+ DeconstructQualifiedName(qualified_format, &schema, &fmt);
+ if (!schema || strcmp(schema, "pg_catalog") == 0)
+ {
+ if (strcmp(fmt, "csv") == 0)
+ opts_out->csv_mode = true;
+ else if (strcmp(fmt, "binary") == 0)
+ opts_out->binary = true;
+ }
Non-qualified names depend on the search_path value so it's not
necessarily a built-in format. If the user specifies 'csv' with
seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily
sets csv_mode true. I think we can instead check if the retrieved
handler function's OID matches the built-in formats' ones. Also, it's
weired to me that cstate has csv_mode and binary fields even though
the format should have already been known by the callback functions.
Regarding the documentation for the existing options, it says "...
only when not using XXX format." some places, where XXX can be
replaced with binary or CSV. Once we support custom formats, 'non-CSV
mode' would actually include custom formats as well, so we need to
update the description too.
>
> > ---
> > +static void
> > +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
> > +{
> > + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u",
> > slot->tts_nvalid)));
> > +}
> >
> > Similar to the above comment, the field name 'tts_nvalid' might also
> > be changed in the future, let's use another name.
>
> Hmm. If the field name is changed, we need to change this
> code.
Yes, but if we use independe name in the NOTICE message we would not
need to update the expected files.
>
> > ---
> > +/*
> > + * Export CopySendEndOfRow() for extensions. We want to keep
> > + * CopySendEndOfRow() as a static function for
> > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a
> > + * compiler.
> > + */
> > +void
> > +CopyToStateFlush(CopyToState cstate)
> > +{
> > + CopySendEndOfRow(cstate);
> > +}
> >
> > Is there any reason to use a different name for public functions?
>
> In this patch set, I use "CopyFrom"/"CopyTo" prefixes for
> public APIs for custom COPY FORMAT handler extensions. It
> will help understanding related APIs. Is it strange in
> PostgreSQL?
I see your point. Probably we need to find a better name as the name
CopyToStateFlush doesn't sound well like this API should be called
only once at the end of a row (IOW user might try to call it multiple
times to 'flush' the state while processing a row). How about
CopyToEndOfRow()?
>
> > ---
> > + /* For custom format implementation */
> > + void *opaque; /* private space */
> >
> > How about renaming 'private'?
>
> We should not use "private" because it's a keyword in
> C++. If we use "private" here, we can't include this file
> from C++ code.
Understood.
>
> > ---
> > I've not reviewed the documentation patch yet but I think the patch
> > seems to miss the updates to the description of the FORMAT option in
> > the COPY command section.
>
> I defer this for now. We can revisit the last documentation
> patch after we finalize our API. (Or could someone help us?)
>
> > I think we can reorganize the patch set as follows:
> >
> > 1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and
> > COPY_DEST_XXX accordingly.
> > 2. Support custom format for both COPY TO and COPY FROM.
> > 3. Expose necessary helper functions such as CopySendEndOfRow().
> > 4. Add CopyFromSkipErrorRow().
> > 5. Documentation.
>
> The attached v39 patch set uses the followings:
>
> 0001: Create copyto_internal.h and change COPY_XXX to
> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
> (Same as 1. in your suggestion)
> 0002: Support custom format for both COPY TO and COPY FROM.
> (Same as 2. in your suggestion)
> 0003: Expose necessary helper functions such as CopySendEndOfRow()
> and add CopyFromSkipErrorRow().
> (3. + 4. in your suggestion)
> 0004: Define handler functions for built-in formats.
> (Not included in your suggestion)
> 0005: Documentation. (WIP)
> (Same as 5. in your suggestion)
Can we merge 0002 and 0004?
> We can merge 0001 quickly, right?
I don't think it makes sense to push only 0001 as it's a completely
preliminary patch for subsequent patches. It would be prudent to push
it once other patches are ready too.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-29 05:57 ` David G. Johnston <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-29 05:57 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Friday, March 28, 2025, Masahiko Sawada <[email protected]> wrote:
>
>
> One problem in the following chunk I can see is:
>
> + qualified_format = stringToQualifiedNameList(format, NULL);
> + DeconstructQualifiedName(qualified_format, &schema, &fmt);
> + if (!schema || strcmp(schema, "pg_catalog") == 0)
> + {
> + if (strcmp(fmt, "csv") == 0)
> + opts_out->csv_mode = true;
> + else if (strcmp(fmt, "binary") == 0)
> + opts_out->binary = true;
> + }
>
> Non-qualified names depend on the search_path value so it's not
> necessarily a built-in format. If the user specifies 'csv' with
> seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily
> sets csv_mode true. I think we can instead check if the retrieved
> handler function's OID matches the built-in formats' ones. Also, it's
> weired to me that cstate has csv_mode and binary fields even though
> the format should have already been known by the callback functions.
>
I considered it a feature that a schema-less reference to text, csv, or
binary always resolves to the core built-in handlers. As does an
unspecified format default of text.
To use an extension that chooses to override that format name would require
schema qualification.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-29 08:57 ` Sutou Kouhei <[email protected]>
2025-03-29 16:12 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
1 sibling, 2 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-03-29 08:57 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBKMNsO+b6wahb6847xwFci1JCfV+JykoMziVgiFxB6cw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 28 Mar 2025 22:37:03 -0700,
Masahiko Sawada <[email protected]> wrote:
>> I've added the following tests:
>>
>> * Wrong input type handler without namespace
>> * Wrong input type handler with namespace
>> * Wrong return type handler without namespace
>> * Wrong return type handler with namespace
>> * Wrong return value (Copy*Routine isn't returned) handler without namespace
>> * Wrong return value (Copy*Routine isn't returned) handler with namespace
>> * Nonexistent handler
>> * Invalid qualified name
>> * Valid handler without namespace and without search_path
>> * Valid handler without namespace and with search_path
>> * Valid handler with namespace
>
> Probably we can merge these newly added four files into one .sql file?
I know that src/test/regress/sql/ uses this style (one .sql
file includes many test patterns in one large category). I
understand that the style is preferable in
src/test/regress/sql/ because src/test/regress/sql/ has
tests for many categories.
But do we need to follow the style in
src/test/modules/*/sql/ too? If we use the style in
src/test/modules/*/sql/, we need to have only one .sql in
src/test/modules/*/sql/ because src/test/modules/*/ are for
each category.
And the current .sql per sub-category style is easy to debug
(at least for me). For example, if we try qualified name
cases on debugger, we can use "\i sql/schema.sql" instead of
extracting target statements from .sql that includes many
unrelated statements. (Or we can use "\i sql/all.sql" and
many GDB "continue"s.)
BTW, it seems that src/test/modules/test_ddl_deparse/sql/
uses .sql per sub-category style. Should we use one .sql
file for sql/test/modules/test_copy_format/sql/? If it's
required for merging this patch set, I'll do it.
> Also we need to add some comments to describe what these queries test.
> For example, it's not clear to me at a glance what queries in
> no-schema.sql are going to test as there is no comment there.
Hmm. You refer no_schema.sql in 0002, right?
----
CREATE EXTENSION test_copy_format;
CREATE TABLE public.test (a smallint, b integer, c bigint);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
\.
COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
DROP TABLE public.test;
DROP EXTENSION test_copy_format;
----
In general, COPY FORMAT tests focus on "COPY FROM WITH
(FORMAT ...)" and "COPY TO WITH (FORMAT ...)". And the file
name "no_schema" shows that it doesn't use qualified
name. Based on this, I feel that the above content is very
straightforward without any comment.
What should we add as comments? For example, do we need the
following comments?
----
-- This extension includes custom COPY handler: test_copy_format
CREATE EXTENSION test_copy_format;
-- Test data
CREATE TABLE public.test (a smallint, b integer, c bigint);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
-- Use custom COPY handler, test_copy_format, without
-- schema for FROM.
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
\.
-- Use custom COPY handler, test_copy_format, without
-- schema for TO.
COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
-- Cleanup
DROP TABLE public.test;
DROP EXTENSION test_copy_format;
----
> One problem in the following chunk I can see is:
>
> + qualified_format = stringToQualifiedNameList(format, NULL);
> + DeconstructQualifiedName(qualified_format, &schema, &fmt);
> + if (!schema || strcmp(schema, "pg_catalog") == 0)
> + {
> + if (strcmp(fmt, "csv") == 0)
> + opts_out->csv_mode = true;
> + else if (strcmp(fmt, "binary") == 0)
> + opts_out->binary = true;
> + }
>
> Non-qualified names depend on the search_path value so it's not
> necessarily a built-in format. If the user specifies 'csv' with
> seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily
> sets csv_mode true.
I think that we should always use built-in COPY handlers for
(not-qualified) "text", "csv" and "binary" for
compatibility. If we allow custom COPY handlers for
(not-qualified) "text", "csv" and "binary", pg_dump or
existing dump may be broken. Because we must use the same
COPY handler when we dump (COPY TO) and we restore (COPY
FROM).
BTW, the current implementation always uses
pg_catalog.{text,csv,binary} for (not-qualified) "text",
"csv" and "binary" even when there are
myschema.{text,csv,binary}. See
src/test/modules/test_copy_format/sql/builtin.sql. But I
haven't looked into it why...
> I think we can instead check if the retrieved
> handler function's OID matches the built-in formats' ones.
I agree that the approach is clear than the current
implementation. I'll use it when I create the next patch
set.
> Also, it's
> weired to me that cstate has csv_mode and binary fields even though
> the format should have already been known by the callback functions.
You refer CopyFomratOptions::{csv_mode,binary} not
Copy{To,From}StateData, right? And you suggest that we
should replace all opts.csv_mode and opts.binary with
opts.handler == F_CSV and opts.handler == F_BINARY, right?
We can do it but I suggest that we do it as a refactoring
(or cleanup) in a separated patch for easy to review.
> Regarding the documentation for the existing options, it says "...
> only when not using XXX format." some places, where XXX can be
> replaced with binary or CSV. Once we support custom formats, 'non-CSV
> mode' would actually include custom formats as well, so we need to
> update the description too.
I agree with you.
>> > ---
>> > +/*
>> > + * Export CopySendEndOfRow() for extensions. We want to keep
>> > + * CopySendEndOfRow() as a static function for
>> > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a
>> > + * compiler.
>> > + */
>> > +void
>> > +CopyToStateFlush(CopyToState cstate)
>> > +{
>> > + CopySendEndOfRow(cstate);
>> > +}
>> >
>> > Is there any reason to use a different name for public functions?
>>
>> In this patch set, I use "CopyFrom"/"CopyTo" prefixes for
>> public APIs for custom COPY FORMAT handler extensions. It
>> will help understanding related APIs. Is it strange in
>> PostgreSQL?
>
> I see your point. Probably we need to find a better name as the name
> CopyToStateFlush doesn't sound well like this API should be called
> only once at the end of a row (IOW user might try to call it multiple
> times to 'flush' the state while processing a row). How about
> CopyToEndOfRow()?
CopyToStateFlush() can be called multiple times in a row. It
can also be called only once with multiple rows. Because it
just flushes the current buffer.
Existing CopySendEndOfRow() is called at the end of a
row. (Buffer is flushed at the end of row.) So I think that
the "EndOfRow" was chosen.
Some custom COPY handlers may not be row based. For example,
Apache Arrow COPY handler doesn't flush buffer for each row.
So, we should provide "flush" API not "end of row" API for
extensibility.
>> The attached v39 patch set uses the followings:
>>
>> 0001: Create copyto_internal.h and change COPY_XXX to
>> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
>> (Same as 1. in your suggestion)
>> 0002: Support custom format for both COPY TO and COPY FROM.
>> (Same as 2. in your suggestion)
>> 0003: Expose necessary helper functions such as CopySendEndOfRow()
>> and add CopyFromSkipErrorRow().
>> (3. + 4. in your suggestion)
>> 0004: Define handler functions for built-in formats.
>> (Not included in your suggestion)
>> 0005: Documentation. (WIP)
>> (Same as 5. in your suggestion)
>
> Can we merge 0002 and 0004?
Can we do it when we merge this patch set if it's still
desirable at the time? Because:
* I think that separated 0002 and 0004 patches are easier to
review than squashed 0002 and 0004 patch.
* I still think that someone may don't like defining COPY
handlers for built-in formats. If we don't define COPY
handlers for built-in formats finally, we can just drop
0004.
>> We can merge 0001 quickly, right?
>
> I don't think it makes sense to push only 0001 as it's a completely
> preliminary patch for subsequent patches. It would be prudent to push
> it once other patches are ready too.
Hmm. I feel that 0001 is a refactoring category patch like
merged patches. In general, distinct enum value names are
easier to understand.
BTW, does the "other patches" include the documentation
patch...?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-29 16:12 ` David G. Johnston <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-29 16:12 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Sat, Mar 29, 2025 at 1:57 AM Sutou Kouhei <[email protected]> wrote:
> * I still think that someone may don't like defining COPY
> handlers for built-in formats. If we don't define COPY
> handlers for built-in formats finally, we can just drop
> 0004.
>
We should (and usually do) dog-food APIs when reasonable and this situation
seems quite reasonable. I'd push back quite a bit about publishing this
without any internal code leveraging it.
> >> We can merge 0001 quickly, right?
> >
> > I don't think it makes sense to push only 0001 as it's a completely
> > preliminary patch for subsequent patches. It would be prudent to push
> > it once other patches are ready too.
>
> Hmm. I feel that 0001 is a refactoring category patch like
> merged patches. In general, distinct enum value names are
> easier to understand.
>
>
I'm for pushing 0001. We've had copyfrom_internal.h for a while now and
this seems like a simple refactor to make that area of the code cleaner via
symmetry.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-31 19:35 ` Masahiko Sawada <[email protected]>
2025-05-02 22:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
1 sibling, 3 replies; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-31 19:35 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Sat, Mar 29, 2025 at 1:57 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBKMNsO+b6wahb6847xwFci1JCfV+JykoMziVgiFxB6cw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 28 Mar 2025 22:37:03 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> I've added the following tests:
> >>
> >> * Wrong input type handler without namespace
> >> * Wrong input type handler with namespace
> >> * Wrong return type handler without namespace
> >> * Wrong return type handler with namespace
> >> * Wrong return value (Copy*Routine isn't returned) handler without namespace
> >> * Wrong return value (Copy*Routine isn't returned) handler with namespace
> >> * Nonexistent handler
> >> * Invalid qualified name
> >> * Valid handler without namespace and without search_path
> >> * Valid handler without namespace and with search_path
> >> * Valid handler with namespace
> >
> > Probably we can merge these newly added four files into one .sql file?
>
> I know that src/test/regress/sql/ uses this style (one .sql
> file includes many test patterns in one large category). I
> understand that the style is preferable in
> src/test/regress/sql/ because src/test/regress/sql/ has
> tests for many categories.
>
> But do we need to follow the style in
> src/test/modules/*/sql/ too? If we use the style in
> src/test/modules/*/sql/, we need to have only one .sql in
> src/test/modules/*/sql/ because src/test/modules/*/ are for
> each category.
>
> And the current .sql per sub-category style is easy to debug
> (at least for me). For example, if we try qualified name
> cases on debugger, we can use "\i sql/schema.sql" instead of
> extracting target statements from .sql that includes many
> unrelated statements. (Or we can use "\i sql/all.sql" and
> many GDB "continue"s.)
>
> BTW, it seems that src/test/modules/test_ddl_deparse/sql/
> uses .sql per sub-category style. Should we use one .sql
> file for sql/test/modules/test_copy_format/sql/? If it's
> required for merging this patch set, I'll do it.
I'm not sure that the regression test queries are categorized in the
same way as in test_ddl_deparse. While the former have separate .sql
files for different types of inputs (valid inputs and invalid inputs
etc.) , which seems finer graind, the latter has .sql files for each
DDL command.
Most of the queries under test_copy_format/sql verifies the input
patterns of the FORMAT option. I find that the regression tests
included in that directory probably should focus on testing new
callback APIs and some regression tests for FORMAT option handling can
be moved into the normal regression test suite (e.g., in copy.sql or a
new file like copy_format.sql). IIUC testing for invalid input
patterns can be done even without creating artificial wrong handler
functions.
>
> > Also we need to add some comments to describe what these queries test.
> > For example, it's not clear to me at a glance what queries in
> > no-schema.sql are going to test as there is no comment there.
>
> Hmm. You refer no_schema.sql in 0002, right?
>
> ----
> CREATE EXTENSION test_copy_format;
> CREATE TABLE public.test (a smallint, b integer, c bigint);
> INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
> COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
> \.
> COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
> DROP TABLE public.test;
> DROP EXTENSION test_copy_format;
> ----
>
> In general, COPY FORMAT tests focus on "COPY FROM WITH
> (FORMAT ...)" and "COPY TO WITH (FORMAT ...)". And the file
> name "no_schema" shows that it doesn't use qualified
> name. Based on this, I feel that the above content is very
> straightforward without any comment.
>
> What should we add as comments? For example, do we need the
> following comments?
>
> ----
> -- This extension includes custom COPY handler: test_copy_format
> CREATE EXTENSION test_copy_format;
> -- Test data
> CREATE TABLE public.test (a smallint, b integer, c bigint);
> INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
> -- Use custom COPY handler, test_copy_format, without
> -- schema for FROM.
> COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
> \.
> -- Use custom COPY handler, test_copy_format, without
> -- schema for TO.
> COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
> -- Cleanup
> DROP TABLE public.test;
> DROP EXTENSION test_copy_format;
> ----
I'd like to see in the comment what the tests expect. Taking the
following queries as an example,
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format');
\.
COPY public.test TO stdout WITH (FORMAT 'test_copy_format');
it would help readers understand the test case better if we have a
comment like for example:
-- Specify the custom format name without schema. We test if both
-- COPY TO and COPY FROM can find the correct handler function
-- in public schema.
>
> > One problem in the following chunk I can see is:
> >
> > + qualified_format = stringToQualifiedNameList(format, NULL);
> > + DeconstructQualifiedName(qualified_format, &schema, &fmt);
> > + if (!schema || strcmp(schema, "pg_catalog") == 0)
> > + {
> > + if (strcmp(fmt, "csv") == 0)
> > + opts_out->csv_mode = true;
> > + else if (strcmp(fmt, "binary") == 0)
> > + opts_out->binary = true;
> > + }
> >
> > Non-qualified names depend on the search_path value so it's not
> > necessarily a built-in format. If the user specifies 'csv' with
> > seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily
> > sets csv_mode true.
>
> I think that we should always use built-in COPY handlers for
> (not-qualified) "text", "csv" and "binary" for
> compatibility. If we allow custom COPY handlers for
> (not-qualified) "text", "csv" and "binary", pg_dump or
> existing dump may be broken. Because we must use the same
> COPY handler when we dump (COPY TO) and we restore (COPY
> FROM).
I agreed.
>
> BTW, the current implementation always uses
> pg_catalog.{text,csv,binary} for (not-qualified) "text",
> "csv" and "binary" even when there are
> myschema.{text,csv,binary}. See
> src/test/modules/test_copy_format/sql/builtin.sql. But I
> haven't looked into it why...
Sorry, I don't follow that. IIUC test_copy_format extension doesn't
create a handler function in myschema schema, and SQLs in builtin.sql
seem to work as expected (specifying a non-qualified built-in format
unconditionally uses the built-in format).
>
> > Also, it's
> > weired to me that cstate has csv_mode and binary fields even though
> > the format should have already been known by the callback functions.
>
> You refer CopyFomratOptions::{csv_mode,binary} not
> Copy{To,From}StateData, right?
Yes. I referred to the wrong one.
> And you suggest that we
> should replace all opts.csv_mode and opts.binary with
> opts.handler == F_CSV and opts.handler == F_BINARY, right?
>
> We can do it but I suggest that we do it as a refactoring
> (or cleanup) in a separated patch for easy to review.
I think that csv_mode and binary are used mostly in
ProcessCopyOptions() so probably we can use local variables for that.
I find there are two other places where to use csv_mode:
NextCopyFromRawFields() and CopyToTextLikeStart(). I think we can
simply check the handler function's OID there, or we can define macros
like COPY_FORMAT_IS_TEXT/CSV/BINARY checking the OID and use them
there.
>
> >> > ---
> >> > +/*
> >> > + * Export CopySendEndOfRow() for extensions. We want to keep
> >> > + * CopySendEndOfRow() as a static function for
> >> > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a
> >> > + * compiler.
> >> > + */
> >> > +void
> >> > +CopyToStateFlush(CopyToState cstate)
> >> > +{
> >> > + CopySendEndOfRow(cstate);
> >> > +}
> >> >
> >> > Is there any reason to use a different name for public functions?
> >>
> >> In this patch set, I use "CopyFrom"/"CopyTo" prefixes for
> >> public APIs for custom COPY FORMAT handler extensions. It
> >> will help understanding related APIs. Is it strange in
> >> PostgreSQL?
> >
> > I see your point. Probably we need to find a better name as the name
> > CopyToStateFlush doesn't sound well like this API should be called
> > only once at the end of a row (IOW user might try to call it multiple
> > times to 'flush' the state while processing a row). How about
> > CopyToEndOfRow()?
>
> CopyToStateFlush() can be called multiple times in a row. It
> can also be called only once with multiple rows. Because it
> just flushes the current buffer.
>
> Existing CopySendEndOfRow() is called at the end of a
> row. (Buffer is flushed at the end of row.) So I think that
> the "EndOfRow" was chosen.
>
> Some custom COPY handlers may not be row based. For example,
> Apache Arrow COPY handler doesn't flush buffer for each row.
> So, we should provide "flush" API not "end of row" API for
> extensibility.
Okay, understood.
>
> >> We can merge 0001 quickly, right?
> >
> > I don't think it makes sense to push only 0001 as it's a completely
> > preliminary patch for subsequent patches. It would be prudent to push
> > it once other patches are ready too.
>
> Hmm. I feel that 0001 is a refactoring category patch like
> merged patches. In general, distinct enum value names are
> easier to understand.
Right, but the patches that have already been merged contributed to
speed up COPY commands, but 0001 patch also introduces
copyto_internal.h, which is not used by anyone in a case where the
custom copy format patch is not merged. Without adding
copyto_internal.h changing enum value names less makes sense to me.
> BTW, does the "other patches" include the documentation
> patch...?
I think that when pushing the main custom COPY format patch, we need
to include the documentation changes into it.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-02 22:52 ` Masahiko Sawada <[email protected]>
2025-05-03 02:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-02 22:52 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, May 1, 2025 at 4:04 PM Michael Paquier <[email protected]> wrote:
>
> On Thu, May 01, 2025 at 12:15:30PM -0700, Masahiko Sawada wrote:
> > In light of these concerns, I've been contemplating alternative
> > interface designs. One promising approach would involve registering
> > custom copy formats via a C function during module loading
> > (specifically, in _PG_init()). This method would require extension
> > authors to invoke a registration function, say
> > RegisterCustomCopyFormat(), in _PG_init() as follows:
> >
> > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines",
> > &JsonLinesCopyToRoutine,
> > &JsonLinesCopyFromRoutine);
> >
> > The registration function would validate the format name and store it
> > in TopMemoryContext. It would then return a unique identifier that can
> > be used subsequently to reference the custom copy format extension.
>
> Hmm. How much should we care about the observability of the COPY
> format used by a given backend? Storing this information in a
> backend's TopMemoryContext is OK to get the extensibility basics to
> work, but could it make sense to use some shmem state to allocate a
> uint32 ID that could be shared by all backends. Contrary to EXPLAIN,
> COPY commands usually run for a very long time, so I am wondering if
> these APIs should be designed so as it would be possible to monitor
> the format used. One layer where the format information could be made
> available is the progress reporting view for COPY, for example. I can
> also imagine a pgstats kind where we do COPY stats aggregates, with a
> per-format pgstats kind, and sharing a fixed ID across multiple
> backends is relevant (when flushing the stats at shutdown, we would
> use a name/ID mapping like replication slots).
>
> I don't think that this needs to be relevant for the option part, just
> for the format where, I suspect, we should store in a shmem array
> based on the ID allocated the name of the format, the library of the
> callback and the function name fed to load_external_function().
>
> Note that custom LWLock and wait events use a shmem state for
> monitoring purposes, where we are able to do ID->format name lookups
> as much as format->ID lookups. Perhaps it's OK not to do that for
> COPY, but I am wondering if we'd better design things from scratch
> with states in shmem state knowing that COPY is a long-running
> operation, and that if one mixes multiple formats they would most
> likely want to know which formats are bottlenecks, through SQL. Cloud
> providers would love that.
Good point. It would make sense to have such information as a map on
shmem. It might be better to use dshash here since a custom copy
format module can be loaded at runtime. Or we can use dynahash with
large enough elements.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-02 22:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 02:24 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-03 02:24 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoB82+MoP_RJ=zzhO9KaHK4LbfGjORkre34C7g-xsCdegQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 15:52:49 -0700,
Masahiko Sawada <[email protected]> wrote:
>> Hmm. How much should we care about the observability of the COPY
>> format used by a given backend? Storing this information in a
>> backend's TopMemoryContext is OK to get the extensibility basics to
>> work, but could it make sense to use some shmem state to allocate a
>> uint32 ID that could be shared by all backends. Contrary to EXPLAIN,
>> COPY commands usually run for a very long time, so I am wondering if
>> these APIs should be designed so as it would be possible to monitor
>> the format used. One layer where the format information could be made
>> available is the progress reporting view for COPY, for example. I can
>> also imagine a pgstats kind where we do COPY stats aggregates, with a
>> per-format pgstats kind, and sharing a fixed ID across multiple
>> backends is relevant (when flushing the stats at shutdown, we would
>> use a name/ID mapping like replication slots).
>>
>> I don't think that this needs to be relevant for the option part, just
>> for the format where, I suspect, we should store in a shmem array
>> based on the ID allocated the name of the format, the library of the
>> callback and the function name fed to load_external_function().
>>
>> Note that custom LWLock and wait events use a shmem state for
>> monitoring purposes, where we are able to do ID->format name lookups
>> as much as format->ID lookups. Perhaps it's OK not to do that for
>> COPY, but I am wondering if we'd better design things from scratch
>> with states in shmem state knowing that COPY is a long-running
>> operation, and that if one mixes multiple formats they would most
>> likely want to know which formats are bottlenecks, through SQL. Cloud
>> providers would love that.
>
> Good point. It would make sense to have such information as a map on
> shmem. It might be better to use dshash here since a custom copy
> format module can be loaded at runtime. Or we can use dynahash with
> large enough elements.
If we don't need to assign an ID for each format, can we
avoid it? If we implement it, is this approach more complex
than the current table sampling method like approach?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 02:19 ` Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-03 02:19 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBuEqcz2_+dpA3WTiDUF=FgudPBKwM+nvH+qHT-k4p5mA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 1 May 2025 12:15:30 -0700,
Masahiko Sawada <[email protected]> wrote:
> One of the primary considerations we need to address is the treatment
> of the specified format name. The current patch set utilizes built-in
> formats (namely 'csv', 'text', and 'binary') when the format name is
> either unqualified or explicitly specified with 'pg_catalog' as the
> schema. In all other cases, we search for custom format handler
> functions based on the search_path. To be frank, I have reservations
> about this interface design, as the dependence of the specified custom
> format name on the search_path could potentially confuse users.
How about requiring schema for all custom formats?
Valid:
COPY ... TO ... (FORMAT 'text');
COPY ... TO ... (FORMAT 'my_schema.jsonlines');
Invalid:
COPY ... TO ... (FORMAT 'jsonlines'); -- no schema
COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema
If we require "schema" for all custom formats, we don't need
to depend on search_path.
> In light of these concerns, I've been contemplating alternative
> interface designs. One promising approach would involve registering
> custom copy formats via a C function during module loading
> (specifically, in _PG_init()). This method would require extension
> authors to invoke a registration function, say
> RegisterCustomCopyFormat(), in _PG_init() as follows:
>
> JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines",
> &JsonLinesCopyToRoutine,
> &JsonLinesCopyFromRoutine);
>
> The registration function would validate the format name and store it
> in TopMemoryContext. It would then return a unique identifier that can
> be used subsequently to reference the custom copy format extension.
I don't object the suggested interface because I don't have
a strong opinion how to implement this feature.
Why do we need to assign a unique ID? For performance? For
RegisterCustomCopyFormatOption()?
I think that we don't need to use it so much in COPY. We
don't need to use format name and assigned ID after we
retrieve a corresponding Copy{To,From}Routine. Because all
needed information are in Copy{To,From}Routine.
> Extensions could register their own options within this
> framework, for example:
>
> RegisterCustomCopyFormatOption(JsonLinesFormatId,
> "custom_option",
> custom_option_handler);
Can we defer to discuss how to add support for custom
options while we focus on the first implementation? Earlier
patch sets with the current approach had custom options
support but it's removed in the first implementation.
(BTW, I think that it's not a good API because we want COPY
FROM only options and COPY TO only options something like
"compression level".)
> This approach offers several advantages: it would eliminate the
> search_path issue, provide greater flexibility, and potentially
> simplify the overall interface for users and developers alike.
What contributes to the "flexibility"? Developers can call
multiple Register* functions in _PG_Init(), right?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-05-03 04:38 ` Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-03 04:38 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, May 2, 2025 at 7:20 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBuEqcz2_+dpA3WTiDUF=FgudPBKwM+nvH+qHT-k4p5mA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 1 May 2025 12:15:30 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > One of the primary considerations we need to address is the treatment
> > of the specified format name. The current patch set utilizes built-in
> > formats (namely 'csv', 'text', and 'binary') when the format name is
> > either unqualified or explicitly specified with 'pg_catalog' as the
> > schema. In all other cases, we search for custom format handler
> > functions based on the search_path. To be frank, I have reservations
> > about this interface design, as the dependence of the specified custom
> > format name on the search_path could potentially confuse users.
>
> How about requiring schema for all custom formats?
>
> Valid:
>
> COPY ... TO ... (FORMAT 'text');
> COPY ... TO ... (FORMAT 'my_schema.jsonlines');
>
> Invalid:
>
> COPY ... TO ... (FORMAT 'jsonlines'); -- no schema
> COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema
>
> If we require "schema" for all custom formats, we don't need
> to depend on search_path.
I'm concerned that users cannot use the same format name in the FORMAT
option depending on which schema the handler function is created.
>
> > In light of these concerns, I've been contemplating alternative
> > interface designs. One promising approach would involve registering
> > custom copy formats via a C function during module loading
> > (specifically, in _PG_init()). This method would require extension
> > authors to invoke a registration function, say
> > RegisterCustomCopyFormat(), in _PG_init() as follows:
> >
> > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines",
> > &JsonLinesCopyToRoutine,
> > &JsonLinesCopyFromRoutine);
> >
> > The registration function would validate the format name and store it
> > in TopMemoryContext. It would then return a unique identifier that can
> > be used subsequently to reference the custom copy format extension.
>
> I don't object the suggested interface because I don't have
> a strong opinion how to implement this feature.
>
> Why do we need to assign a unique ID? For performance? For
> RegisterCustomCopyFormatOption()?
I think it's required for monitoring purposes for example. For
instance, we can set the format ID in the progress information and the
progress view can fetch the format name by the ID so that users can
see what format is being used in the COPY command.
>
> > Extensions could register their own options within this
> > framework, for example:
> >
> > RegisterCustomCopyFormatOption(JsonLinesFormatId,
> > "custom_option",
> > custom_option_handler);
>
> Can we defer to discuss how to add support for custom
> options while we focus on the first implementation? Earlier
> patch sets with the current approach had custom options
> support but it's removed in the first implementation.
I think we can skip the custom option patch for the first
implementation but still need to discuss how we will be able to
implement it to understand the big picture of this feature. Otherwise
we could end up going the wrong direction.
>
> (BTW, I think that it's not a good API because we want COPY
> FROM only options and COPY TO only options something like
> "compression level".)
Why does this matter in terms of API? I think that even with this API
we can pass is_from to the option handler function so that it
validates the option based on it.
>
> > This approach offers several advantages: it would eliminate the
> > search_path issue, provide greater flexibility, and potentially
> > simplify the overall interface for users and developers alike.
>
> What contributes to the "flexibility"? Developers can call
> multiple Register* functions in _PG_Init(), right?
I think that with a tablesample-like approach we need to do everything
based on one handler function and callbacks returned from it whereas
there is no such limitation with C API style.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 04:56 ` Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-03 04:56 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBGRFStdVbHUcxL0QB8wn92J3Sn-6x=RhsSMuhepRH0NQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 21:38:32 -0700,
Masahiko Sawada <[email protected]> wrote:
>> How about requiring schema for all custom formats?
>>
>> Valid:
>>
>> COPY ... TO ... (FORMAT 'text');
>> COPY ... TO ... (FORMAT 'my_schema.jsonlines');
>>
>> Invalid:
>>
>> COPY ... TO ... (FORMAT 'jsonlines'); -- no schema
>> COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema
>>
>> If we require "schema" for all custom formats, we don't need
>> to depend on search_path.
>
> I'm concerned that users cannot use the same format name in the FORMAT
> option depending on which schema the handler function is created.
I'm not sure that it's a problem or not. If users want to
use the same format name, they can install the handler
function to the same schema.
>> Why do we need to assign a unique ID? For performance? For
>> RegisterCustomCopyFormatOption()?
>
> I think it's required for monitoring purposes for example. For
> instance, we can set the format ID in the progress information and the
> progress view can fetch the format name by the ID so that users can
> see what format is being used in the COPY command.
How about setting the format name instead of the format ID
in the progress information?
> I think we can skip the custom option patch for the first
> implementation but still need to discuss how we will be able to
> implement it to understand the big picture of this feature. Otherwise
> we could end up going the wrong direction.
I think that we don't need to discuss it deeply because we
have many options with this approach. We can call C
functions in _PG_Init(). I think that this feature will not
be a blocker of this approach.
>> (BTW, I think that it's not a good API because we want COPY
>> FROM only options and COPY TO only options something like
>> "compression level".)
>
> Why does this matter in terms of API? I think that even with this API
> we can pass is_from to the option handler function so that it
> validates the option based on it.
If we choose the API, each custom format developer needs to
handle the case in handler function. For example, if we pass
information whether this option is only for TO to
PostgreSQL, ProcessCopyOptions() not handler functions can
handle it.
Anyway, I think that we don't need to discuss this deeply
for now.
>> What contributes to the "flexibility"? Developers can call
>> multiple Register* functions in _PG_Init(), right?
>
> I think that with a tablesample-like approach we need to do everything
> based on one handler function and callbacks returned from it whereas
> there is no such limitation with C API style.
Thanks for clarifying it. It seems that my understanding is
correct.
I hope that the flexibility is needed flexibility and too
much flexibility doesn't introduce too much complexity.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-05-03 06:02 ` Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-03 06:02 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, May 2, 2025 at 9:56 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBGRFStdVbHUcxL0QB8wn92J3Sn-6x=RhsSMuhepRH0NQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 21:38:32 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> How about requiring schema for all custom formats?
> >>
> >> Valid:
> >>
> >> COPY ... TO ... (FORMAT 'text');
> >> COPY ... TO ... (FORMAT 'my_schema.jsonlines');
> >>
> >> Invalid:
> >>
> >> COPY ... TO ... (FORMAT 'jsonlines'); -- no schema
> >> COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema
> >>
> >> If we require "schema" for all custom formats, we don't need
> >> to depend on search_path.
> >
> > I'm concerned that users cannot use the same format name in the FORMAT
> > option depending on which schema the handler function is created.
>
> I'm not sure that it's a problem or not. If users want to
> use the same format name, they can install the handler
> function to the same schema.
>
> >> Why do we need to assign a unique ID? For performance? For
> >> RegisterCustomCopyFormatOption()?
> >
> > I think it's required for monitoring purposes for example. For
> > instance, we can set the format ID in the progress information and the
> > progress view can fetch the format name by the ID so that users can
> > see what format is being used in the COPY command.
>
> How about setting the format name instead of the format ID
> in the progress information?
The progress view can know only numbers. We need to extend the
progress view infrastructure so that we can pass other data types.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 06:20 ` Sutou Kouhei <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-03 06:20 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoDnY2fhC7tp7jpn24AuwkeW-0YjFEtZbEfPwg8YcH6bAw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:02:25 -0700,
Masahiko Sawada <[email protected]> wrote:
> The progress view can know only numbers. We need to extend the
> progress view infrastructure so that we can pass other data types.
Sorry. Could you tell me what APIs referred here?
pgstat_progress_*() functions in
src/include/utils/backend_progress.h?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-05-03 06:37 ` Masahiko Sawada <[email protected]>
2025-05-09 08:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-03 06:37 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, May 2, 2025 at 11:20 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoDnY2fhC7tp7jpn24AuwkeW-0YjFEtZbEfPwg8YcH6bAw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:02:25 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > The progress view can know only numbers. We need to extend the
> > progress view infrastructure so that we can pass other data types.
>
> Sorry. Could you tell me what APIs referred here?
> pgstat_progress_*() functions in
> src/include/utils/backend_progress.h?
The progress information is stored in PgBackendStatus defined in
backend_status.h:
/*
* Command progress reporting. Any command which wishes can advertise
* that it is running by setting st_progress_command,
* st_progress_command_target, and st_progress_param[].
* st_progress_command_target should be the OID of the relation which the
* command targets (we assume there's just one, as this is meant for
* utility commands), but the meaning of each element in the
* st_progress_param array is command-specific.
*/
ProgressCommandType st_progress_command;
Oid st_progress_command_target;
int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM];
Then the progress view maps the numbers to the corresponding strings:
CREATE VIEW pg_stat_progress_copy AS
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
CASE S.param5 WHEN 1 THEN 'COPY FROM'
WHEN 2 THEN 'COPY TO'
END AS command,
CASE S.param6 WHEN 1 THEN 'FILE'
WHEN 2 THEN 'PROGRAM'
WHEN 3 THEN 'PIPE'
WHEN 4 THEN 'CALLBACK'
END AS "type",
S.param1 AS bytes_processed,
S.param2 AS bytes_total,
S.param3 AS tuples_processed,
S.param4 AS tuples_excluded,
S.param7 AS tuples_skipped
FROM pg_stat_get_progress_info('COPY') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
So the idea is that the backend process sets the format ID somewhere
in st_progress_param, and then the progress view calls a SQL function,
say pg_stat_get_copy_format_name(), with the format ID that returns
the corresponding format name.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-09 08:51 ` Sutou Kouhei <[email protected]>
2025-05-10 04:29 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-09 08:51 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoD9CBjh4u6jdiE0tG-jvejw-GJN8fUPoQSVhKh36HW2NQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:37:46 -0700,
Masahiko Sawada <[email protected]> wrote:
> The progress information is stored in PgBackendStatus defined in
> backend_status.h:
>
> /*
> * Command progress reporting. Any command which wishes can advertise
> * that it is running by setting st_progress_command,
> * st_progress_command_target, and st_progress_param[].
> * st_progress_command_target should be the OID of the relation which the
> * command targets (we assume there's just one, as this is meant for
> * utility commands), but the meaning of each element in the
> * st_progress_param array is command-specific.
> */
> ProgressCommandType st_progress_command;
> Oid st_progress_command_target;
> int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM];
>
> Then the progress view maps the numbers to the corresponding strings:
>
> CREATE VIEW pg_stat_progress_copy AS
> SELECT
> S.pid AS pid, S.datid AS datid, D.datname AS datname,
> S.relid AS relid,
> CASE S.param5 WHEN 1 THEN 'COPY FROM'
> WHEN 2 THEN 'COPY TO'
> END AS command,
> CASE S.param6 WHEN 1 THEN 'FILE'
> WHEN 2 THEN 'PROGRAM'
> WHEN 3 THEN 'PIPE'
> WHEN 4 THEN 'CALLBACK'
> END AS "type",
> S.param1 AS bytes_processed,
> S.param2 AS bytes_total,
> S.param3 AS tuples_processed,
> S.param4 AS tuples_excluded,
> S.param7 AS tuples_skipped
> FROM pg_stat_get_progress_info('COPY') AS S
> LEFT JOIN pg_database D ON S.datid = D.oid;
Thanks. I didn't know about how to implement
pg_stat_progress_copy.
> So the idea is that the backend process sets the format ID somewhere
> in st_progress_param, and then the progress view calls a SQL function,
> say pg_stat_get_copy_format_name(), with the format ID that returns
> the corresponding format name.
Does it work when we use session_preload_libraries or the
LOAD command? If we have 2 sessions and both of them load
"jsonlines" COPY FORMAT extensions, what will be happened?
For example:
1. Session 1: Register "jsonlines"
2. Session 2: Register "jsonlines"
(Should global format ID <-> format name mapping
be updated?)
3. Session 2: Close this session.
Unregister "jsonlines".
(Can we unregister COPY FORMAT extension?)
(Should global format ID <-> format name mapping
be updated?)
4. Session 1: Close this session.
Unregister "jsonlines".
(Can we unregister COPY FORMAT extension?)
(Should global format ID <-> format name mapping
be updated?)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-09 08:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-05-10 04:29 ` Masahiko Sawada <[email protected]>
2025-05-26 01:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-10 04:29 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, May 9, 2025 at 1:51 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoD9CBjh4u6jdiE0tG-jvejw-GJN8fUPoQSVhKh36HW2NQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:37:46 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > The progress information is stored in PgBackendStatus defined in
> > backend_status.h:
> >
> > /*
> > * Command progress reporting. Any command which wishes can advertise
> > * that it is running by setting st_progress_command,
> > * st_progress_command_target, and st_progress_param[].
> > * st_progress_command_target should be the OID of the relation which the
> > * command targets (we assume there's just one, as this is meant for
> > * utility commands), but the meaning of each element in the
> > * st_progress_param array is command-specific.
> > */
> > ProgressCommandType st_progress_command;
> > Oid st_progress_command_target;
> > int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM];
> >
> > Then the progress view maps the numbers to the corresponding strings:
> >
> > CREATE VIEW pg_stat_progress_copy AS
> > SELECT
> > S.pid AS pid, S.datid AS datid, D.datname AS datname,
> > S.relid AS relid,
> > CASE S.param5 WHEN 1 THEN 'COPY FROM'
> > WHEN 2 THEN 'COPY TO'
> > END AS command,
> > CASE S.param6 WHEN 1 THEN 'FILE'
> > WHEN 2 THEN 'PROGRAM'
> > WHEN 3 THEN 'PIPE'
> > WHEN 4 THEN 'CALLBACK'
> > END AS "type",
> > S.param1 AS bytes_processed,
> > S.param2 AS bytes_total,
> > S.param3 AS tuples_processed,
> > S.param4 AS tuples_excluded,
> > S.param7 AS tuples_skipped
> > FROM pg_stat_get_progress_info('COPY') AS S
> > LEFT JOIN pg_database D ON S.datid = D.oid;
>
> Thanks. I didn't know about how to implement
> pg_stat_progress_copy.
>
> > So the idea is that the backend process sets the format ID somewhere
> > in st_progress_param, and then the progress view calls a SQL function,
> > say pg_stat_get_copy_format_name(), with the format ID that returns
> > the corresponding format name.
>
> Does it work when we use session_preload_libraries or the
> LOAD command? If we have 2 sessions and both of them load
> "jsonlines" COPY FORMAT extensions, what will be happened?
>
> For example:
>
> 1. Session 1: Register "jsonlines"
> 2. Session 2: Register "jsonlines"
> (Should global format ID <-> format name mapping
> be updated?)
> 3. Session 2: Close this session.
> Unregister "jsonlines".
> (Can we unregister COPY FORMAT extension?)
> (Should global format ID <-> format name mapping
> be updated?)
> 4. Session 1: Close this session.
> Unregister "jsonlines".
> (Can we unregister COPY FORMAT extension?)
> (Should global format ID <-> format name mapping
> be updated?)
I imagine that only for progress reporting purposes, I think session 1
and 2 can have different format IDs for the same 'jsonlines' if they
load it by LOAD command. They can advertise the format IDs on the
shmem and we can also provide a SQL function for the progress view
that can get the format name by the format ID.
Considering the possibility that we might want to use the format ID
also in the cumulative statistics, we might want to strictly provide
the unique format ID for each custom format as the format IDs are
serialized to the pgstat file. One possible way to implement it is
that we manage the custom format IDs in a wiki page like we do for
custom cumulative statistics and custom RMGR[1][2]. That is, a custom
format extension registers the format name along with the format ID
that is pre-registered in the wiki page or the format ID (e.g. 128)
indicating under development. If either the format name or format ID
conflict with an already registered custom format extension, the
registration function raises an error. And we preallocate enough
format IDs for built-in formats.
As for unregistration, I think that even if we provide an
unregisteration API, it ultimately depends on whether or not custom
format extensions call it in _PG_fini().
Regards,
[1] https://wiki.postgresql.org/wiki/CustomCumulativeStats
[2] https://wiki.postgresql.org/wiki/CustomWALResourceManagers
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-09 08:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 04:29 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-26 01:27 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-26 01:27 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoAY_h-9nuhs14e3cyO_A2rH7==zuq+NPHkn9ggwyaXnPQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 9 May 2025 21:29:23 -0700,
Masahiko Sawada <[email protected]> wrote:
>> > So the idea is that the backend process sets the format ID somewhere
>> > in st_progress_param, and then the progress view calls a SQL function,
>> > say pg_stat_get_copy_format_name(), with the format ID that returns
>> > the corresponding format name.
>>
>> Does it work when we use session_preload_libraries or the
>> LOAD command? If we have 2 sessions and both of them load
>> "jsonlines" COPY FORMAT extensions, what will be happened?
>>
>> For example:
>>
>> 1. Session 1: Register "jsonlines"
>> 2. Session 2: Register "jsonlines"
>> (Should global format ID <-> format name mapping
>> be updated?)
>> 3. Session 2: Close this session.
>> Unregister "jsonlines".
>> (Can we unregister COPY FORMAT extension?)
>> (Should global format ID <-> format name mapping
>> be updated?)
>> 4. Session 1: Close this session.
>> Unregister "jsonlines".
>> (Can we unregister COPY FORMAT extension?)
>> (Should global format ID <-> format name mapping
>> be updated?)
>
> I imagine that only for progress reporting purposes, I think session 1
> and 2 can have different format IDs for the same 'jsonlines' if they
> load it by LOAD command. They can advertise the format IDs on the
> shmem and we can also provide a SQL function for the progress view
> that can get the format name by the format ID.
>
> Considering the possibility that we might want to use the format ID
> also in the cumulative statistics, we might want to strictly provide
> the unique format ID for each custom format as the format IDs are
> serialized to the pgstat file. One possible way to implement it is
> that we manage the custom format IDs in a wiki page like we do for
> custom cumulative statistics and custom RMGR[1][2]. That is, a custom
> format extension registers the format name along with the format ID
> that is pre-registered in the wiki page or the format ID (e.g. 128)
> indicating under development. If either the format name or format ID
> conflict with an already registered custom format extension, the
> registration function raises an error. And we preallocate enough
> format IDs for built-in formats.
>
> As for unregistration, I think that even if we provide an
> unregisteration API, it ultimately depends on whether or not custom
> format extensions call it in _PG_fini().
Thanks for sharing your idea.
With the former ID issuing approach, it seems that we need a
global format ID <-> name mapping and a per session
registered format name list. The custom COPY FORMAT register
function rejects the same format name, right? If we support
both of shared_preload_libraries and
session_preload_libraries/LOAD, we have different life time
custom formats. It may introduce a complexity with the ID
issuing approach.
With the latter static ID approach, how to implement a
function that converts format ID to format name? PostgreSQL
itself doesn't know ID <-> name mapping in the Wiki page. It
seems that custom COPY FORMAT implementation needs to
register its name to PostgreSQL by itself.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 05:36 ` David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2 siblings, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-05-03 05:36 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Thursday, May 1, 2025, Masahiko Sawada <[email protected]> wrote:
>
> In light of these concerns, I've been contemplating alternative
> interface designs. One promising approach would involve registering
> custom copy formats via a C function during module loading
> (specifically, in _PG_init()). This method would require extension
> authors to invoke a registration function, say
> RegisterCustomCopyFormat(), in _PG_init() as follows:
>
> JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines",
> &JsonLinesCopyToRoutine,
> &JsonLinesCopyFromRoutine);
>
> The registration function would validate the format name and store it
> in TopMemoryContext. It would then return a unique identifier that can
> be used subsequently to reference the custom copy format extension.
>
How does this fix the search_path concern? Are query writers supposed to
put JsonLinesFormatId into their queries? Or are you just prohibiting a
DBA from ever installing an extension that wants to register a format name
that is already registered so that no namespace is ever required?
ISTM accommodating a namespace for formats is required just like we do for
virtually every other named object in the system. At least, if we want to
play nice with extension authors. It doesn’t have to be within the
existing pg_proc scope, we can create a new scope if desired, but
abolishing it seems unwise.
It would be more consistent with established policy if we didn’t make
exceptions for text/csv/binary - if the DBA permits a text format to exist
in a different schema and that schema appears first in the search_path,
unqualified references to text would resolve to the non-core handler. We
already protect ourselves with safe search_paths. This is really no
different than if someone wanted to implement a now() function and people
are putting pg_catalog from of existing usage. It’s the DBAs problem, not
ours.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-05-03 05:57 ` Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-03 05:57 UTC (permalink / raw)
To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Fri, May 2, 2025 at 10:36 PM David G. Johnston
<[email protected]> wrote:
>
> On Thursday, May 1, 2025, Masahiko Sawada <[email protected]> wrote:
>>
>>
>> In light of these concerns, I've been contemplating alternative
>> interface designs. One promising approach would involve registering
>> custom copy formats via a C function during module loading
>> (specifically, in _PG_init()). This method would require extension
>> authors to invoke a registration function, say
>> RegisterCustomCopyFormat(), in _PG_init() as follows:
>>
>> JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines",
>> &JsonLinesCopyToRoutine,
>> &JsonLinesCopyFromRoutine);
>>
>> The registration function would validate the format name and store it
>> in TopMemoryContext. It would then return a unique identifier that can
>> be used subsequently to reference the custom copy format extension.
>
>
> How does this fix the search_path concern? Are query writers supposed to put JsonLinesFormatId into their queries? Or are you just prohibiting a DBA from ever installing an extension that wants to register a format name that is already registered so that no namespace is ever required?
Users can specify "jsonlines", passed in the first argument to the
register function, to the COPY FORMAT option in this case. While
JsonLinesFormatId is reserved for internal operations such as module
processing and monitoring, any attempt to load another custom COPY
format module named 'jsonlines' will result in an error.
> ISTM accommodating a namespace for formats is required just like we do for virtually every other named object in the system. At least, if we want to play nice with extension authors. It doesn’t have to be within the existing pg_proc scope, we can create a new scope if desired, but abolishing it seems unwise.
>
> It would be more consistent with established policy if we didn’t make exceptions for text/csv/binary - if the DBA permits a text format to exist in a different schema and that schema appears first in the search_path, unqualified references to text would resolve to the non-core handler. We already protect ourselves with safe search_paths. This is really no different than if someone wanted to implement a now() function and people are putting pg_catalog from of existing usage. It’s the DBAs problem, not ours.
I'm concerned about allowing multiple 'text' format implementations
with identical names within the database, as this could lead to
considerable confusion. When users specify 'text', it would be more
logical to guarantee that the built-in 'text' format is consistently
used. This principle aligns with other customizable components, such
as custom resource managers, wait events, lightweight locks, and
custom scans. These components maintain their built-in data/types and
explicitly prevent the registration of duplicate names.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 06:37 ` David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-05-03 06:37 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Friday, May 2, 2025, Masahiko Sawada <[email protected]> wrote:
>
> I'm concerned about allowing multiple 'text' format implementations
> with identical names within the database, as this could lead to
> considerable confusion. When users specify 'text', it would be more
> logical to guarantee that the built-in 'text' format is consistently
> used.
Do you want to only give text/csv/binary this special treatment or also any
future format name we ever decide to implement in core. If an extension
takes up “xml” and we try to do that in core do we fail an upgrade because
of the conflict, and make it impossible to actually use said extension?
This principle aligns with other customizable components, such
> as custom resource managers, wait events, lightweight locks, and
> custom scans. These components maintain their built-in data/types and
> explicitly prevent the registration of duplicate names.
>
I am totally lost on how any of those resemble this feature.
I’m all for registration to enable additional options and features - but am
against moving away from turning format into a namespaced identifier. This
is a query-facing feature where namespaces are common and fundamentally
required. I have some sympathy for the fact that until now one could not
prefix text/binary/csv with pg_catalog to be fully safe, but in reality
DBAs/query authors either put pg_catalog first in their search_path or make
an informed decision when they deviate. That is the established precedent
relevant to this feature. The power, and responsibility for education,
lies with the user.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-05-03 07:54 ` Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-03 07:54 UTC (permalink / raw)
To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Fri, May 2, 2025 at 11:37 PM David G. Johnston
<[email protected]> wrote:
>
> On Friday, May 2, 2025, Masahiko Sawada <[email protected]> wrote:
>>
>>
>> I'm concerned about allowing multiple 'text' format implementations
>> with identical names within the database, as this could lead to
>> considerable confusion. When users specify 'text', it would be more
>> logical to guarantee that the built-in 'text' format is consistently
>> used.
>
>
> Do you want to only give text/csv/binary this special treatment or also any future format name we ever decide to implement in core. If an extension takes up “xml” and we try to do that in core do we fail an upgrade because of the conflict, and make it impossible to actually use said extension?
I guess that's an extension author's responsibility to upgrade its
extension so as to work with the new PostgreSQL version, or carefully
choose the format name. They can even name
'[extension_name].[format_name]' as a format name. Even with the
current patch design (i.e., search_path affects handler function
lookups), users would end up using the built-in 'xml' format without
notice after upgrade, no? I guess that could introduce another
problem.
I think that we need to ensure that if users specify text/csv/binary
the built-in formats are always used, to keep backward compatibility.
>
>> This principle aligns with other customizable components, such
>> as custom resource managers, wait events, lightweight locks, and
>> custom scans. These components maintain their built-in data/types and
>> explicitly prevent the registration of duplicate names.
>
>
> I am totally lost on how any of those resemble this feature.
>
> I’m all for registration to enable additional options and features - but am against moving away from turning format into a namespaced identifier. This is a query-facing feature where namespaces are common and fundamentally required.
That's a fair concern. But isn't the format name ultimately just an
option value, but not like a database object? As I mentioned above, I
think we need to keep backward compatibility but treating the built-in
formats special seems inconsistent with common name resolution
behavior.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-03 14:42 ` David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-05-03 14:42 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote:
> I think that we need to ensure that if users specify text/csv/binary
> the built-in formats are always used, to keep backward compatibility.
That was my original thinking, but it’s inconsistent with how functions
behave today. We don’t promise that installing extensions won’t cause
existing code to change.
>
>
> > I’m all for registration to enable additional options and features - but
> am against moving away from turning format into a namespaced identifier.
> This is a query-facing feature where namespaces are common and
> fundamentally required.
>
> That's a fair concern. But isn't the format name ultimately just an
> option value, but not like a database object?
We get to decide that. And deciding in favor of “extensible database
object in a namespace’ makes more sense - leveraging all that pre-existing
design to play more nicely with extensions and give DBAs control. The SQL
command to add one is “create function” instead of “create copy format”.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-05-04 05:02 ` Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-04 05:02 UTC (permalink / raw)
To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Sat, May 3, 2025 at 7:42 AM David G. Johnston
<[email protected]> wrote:
>
> On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote:
>
>>
>> I think that we need to ensure that if users specify text/csv/binary
>> the built-in formats are always used, to keep backward compatibility.
>
>
> That was my original thinking, but it’s inconsistent with how functions behave today. We don’t promise that installing extensions won’t cause existing code to change.
I'm skeptical about whether that's an acceptable backward
compatibility breakage.
>> > I’m all for registration to enable additional options and features - but am against moving away from turning format into a namespaced identifier. This is a query-facing feature where namespaces are common and fundamentally required.
>>
>> That's a fair concern. But isn't the format name ultimately just an
>> option value, but not like a database object?
>
>
> We get to decide that. And deciding in favor of “extensible database object in a namespace’ makes more sense - leveraging all that pre-existing design to play more nicely with extensions and give DBAs control. The SQL command to add one is “create function” instead of “create copy format”.
I still don't fully understand why the FORMAT value alone needs to be
treated like a schema-qualified object. If the concern is about name
conflict with future built-in formats, I would argue that the same
concern applies to custom EXPLAIN options and logical decoding
plugins. To me, the benefit of treating the COPY FORMAT value as a
schema-qualified object seems limited. Meanwhile, the risk of not
protecting built-in formats like 'text', 'csv', and 'binary' is
significant. If those names can be shadowed by extension via
search_patch, we lose backward compatibility.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-04 05:27 ` David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-05-04 05:27 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote:
> On Sat, May 3, 2025 at 7:42 AM David G. Johnston
> <[email protected]> wrote:
> >
> > On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote:
> >
> >>
> >> I think that we need to ensure that if users specify text/csv/binary
> >> the built-in formats are always used, to keep backward compatibility.
> >
> >
> > That was my original thinking, but it’s inconsistent with how functions
> behave today. We don’t promise that installing extensions won’t cause
> existing code to change.
>
> I'm skeptical about whether that's an acceptable backward
> compatibility breakage.
I’m skeptical you are correctly defining what backward-compatibility
requires.
Well, the only potential breakage is that we are searching for a matching
function by signature without first limiting the mandated return type. But
that is solve-able should anyone else see the problem as well.
The global format name has its merits but neither it nor the namespaced
format option suffer from breaking compatibility or policy.
>
> I still don't fully understand why the FORMAT value alone needs to be
> treated like a schema-qualified object. If the concern is about name
> conflict with future built-in formats, I would argue that the same
> concern applies to custom EXPLAIN options and logical decoding
> plugins.
>
Then maybe we have the same “problem” in those places.
>
> To me, the benefit of treating the COPY FORMAT value as a
> schema-qualified object seems limited. Meanwhile, the risk of not
> protecting built-in formats like 'text', 'csv', and 'binary' is
> significant.
Really? You think lots of extensions are going to choose to use these
values even if they are permitted? Or are you concerned about attack
surfaces?
> If those names can be shadowed by extension via
> search_patch, we lose backward compatibility.
>
This is not a definition of backward-compatibility that I am familiar with.
If anything the ability for a DBA to arrange for such shadowing would be a
feature enhancement. They can drop-in a more efficient or desirable
implementation without having to change query code.
In any case, I’m doubtful either of us can make a convincing enough
argument to sway the other fully. Both options are plausible, IMO. Others
need to chime in.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-05-09 09:41 ` Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-09 09:41 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAKFQuwaRDXANaL+QcT6LZRAem4rwkSwv9v+viv_mcR+Rex3quA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 3 May 2025 22:27:36 -0700,
"David G. Johnston" <[email protected]> wrote:
> In any case, I’m doubtful either of us can make a convincing enough
> argument to sway the other fully. Both options are plausible, IMO. Others
> need to chime in.
I may misunderstand but here is the current summary, right?
Proposed approaches to register custom COPY formats:
a. Create a function that has the same name of custom COPY
format
b. Call a register function from _PG_init()
FYI: I proposed c. approach that uses a. but it always
requires schema name for format name in other e-mail.
Users can register the same format name:
a. Yes
* Users can distinct the same format name by schema name
* If format name doesn't have schema name, the used
format depends on search_path
* Pros:
* Using schema for it is consistent with other
PostgreSQL mechanisms
* Custom format never conflict with built-in
format. For example, an extension register "xml" and
PostgreSQL adds "xml" later, they are never
conflicted because PostgreSQL's "xml" is registered
to pg_catalog.
* Cons: Different format may be used with the same
input. For example, "jsonlines" may choose
"jsonlines" implemented by extension X or implemented
by extension Y when search_path is different.
b. No
* Users can use "${schema}.${name}" for format name
that mimics PostgreSQL's builtin schema (but it's just
a string)
Built-in formats (text/csv/binary) should be able to
overwritten by extensions:
a. (The current patch is no but David's answer is) Yes
* Pros: Users can use drop-in replacement faster
implementation without changing input
* Cons: Users may overwrite them accidentally.
It may break pg_dump result.
(This is called as "backward incompatibility.")
b. No
Are there any missing or wrong items? If we can summarize
the current discussion here correctly, others will be able
to chime in this discussion. (At least I can do it.)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-05-10 00:57 ` Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-05-10 00:57 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, May 9, 2025 at 2:41 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAKFQuwaRDXANaL+QcT6LZRAem4rwkSwv9v+viv_mcR+Rex3quA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 3 May 2025 22:27:36 -0700,
> "David G. Johnston" <[email protected]> wrote:
>
> > In any case, I’m doubtful either of us can make a convincing enough
> > argument to sway the other fully. Both options are plausible, IMO. Others
> > need to chime in.
>
> I may misunderstand but here is the current summary, right?
Thank you for summarizing the discussion.
>
> Proposed approaches to register custom COPY formats:
> a. Create a function that has the same name of custom COPY
> format
> b. Call a register function from _PG_init()
>
> FYI: I proposed c. approach that uses a. but it always
> requires schema name for format name in other e-mail.
With approach (c), do you mean that we require users to change all
FORMAT option values like from 'text' to 'pg_catalog.text' after the
upgrade? Or are we exempt the built-in formats?
>
> Users can register the same format name:
> a. Yes
> * Users can distinct the same format name by schema name
> * If format name doesn't have schema name, the used
> format depends on search_path
> * Pros:
> * Using schema for it is consistent with other
> PostgreSQL mechanisms
> * Custom format never conflict with built-in
> format. For example, an extension register "xml" and
> PostgreSQL adds "xml" later, they are never
> conflicted because PostgreSQL's "xml" is registered
> to pg_catalog.
> * Cons: Different format may be used with the same
> input. For example, "jsonlines" may choose
> "jsonlines" implemented by extension X or implemented
> by extension Y when search_path is different.
> b. No
> * Users can use "${schema}.${name}" for format name
> that mimics PostgreSQL's builtin schema (but it's just
> a string)
>
>
> Built-in formats (text/csv/binary) should be able to
> overwritten by extensions:
> a. (The current patch is no but David's answer is) Yes
> * Pros: Users can use drop-in replacement faster
> implementation without changing input
> * Cons: Users may overwrite them accidentally.
> It may break pg_dump result.
> (This is called as "backward incompatibility.")
> b. No
The summary matches my understanding. I think the second point is
important. If we go with a tablesample-like API, I agree with David's
point that all FORMAT values including the built-in formats should
depend on the search_path value. While it provides a similar user
experience to other database objects, there is a possibility that a
COPY with built-in format could work differently on v19 than v18 or
earlier depending on the search_path value.
> Are there any missing or wrong items?
I think the approach (b) provides more flexibility than (a) in terms
of API design as with (a) we need to do everything based on one
handler function and callbacks.
> If we can summarize
> the current discussion here correctly, others will be able
> to chime in this discussion. (At least I can do it.)
+1
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-05-26 01:04 ` Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-05-26 01:04 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBrSTmPyDai_QVR-XOe7PL722Dazm70A+FpvGy2hfSV9g@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 9 May 2025 17:57:35 -0700,
Masahiko Sawada <[email protected]> wrote:
>> Proposed approaches to register custom COPY formats:
>> a. Create a function that has the same name of custom COPY
>> format
>> b. Call a register function from _PG_init()
>>
>> FYI: I proposed c. approach that uses a. but it always
>> requires schema name for format name in other e-mail.
>
> With approach (c), do you mean that we require users to change all
> FORMAT option values like from 'text' to 'pg_catalog.text' after the
> upgrade? Or are we exempt the built-in formats?
The latter. 'text' must be accepted because existing pg_dump
results use 'text'. If we reject 'text', it's a big
incompatibility. (We can't dump on old PostgreSQL and
restore to new PostgreSQL.)
>> Users can register the same format name:
>> a. Yes
>> * Users can distinct the same format name by schema name
>> * If format name doesn't have schema name, the used
>> format depends on search_path
>> * Pros:
>> * Using schema for it is consistent with other
>> PostgreSQL mechanisms
>> * Custom format never conflict with built-in
>> format. For example, an extension register "xml" and
>> PostgreSQL adds "xml" later, they are never
>> conflicted because PostgreSQL's "xml" is registered
>> to pg_catalog.
>> * Cons: Different format may be used with the same
>> input. For example, "jsonlines" may choose
>> "jsonlines" implemented by extension X or implemented
>> by extension Y when search_path is different.
>> b. No
>> * Users can use "${schema}.${name}" for format name
>> that mimics PostgreSQL's builtin schema (but it's just
>> a string)
>>
>>
>> Built-in formats (text/csv/binary) should be able to
>> overwritten by extensions:
>> a. (The current patch is no but David's answer is) Yes
>> * Pros: Users can use drop-in replacement faster
>> implementation without changing input
>> * Cons: Users may overwrite them accidentally.
>> It may break pg_dump result.
>> (This is called as "backward incompatibility.")
>> b. No
>
> The summary matches my understanding. I think the second point is
> important. If we go with a tablesample-like API, I agree with David's
> point that all FORMAT values including the built-in formats should
> depend on the search_path value. While it provides a similar user
> experience to other database objects, there is a possibility that a
> COPY with built-in format could work differently on v19 than v18 or
> earlier depending on the search_path value.
Thanks for sharing additional points.
David said that the additional point case is a
responsibility or DBA not PostgreSQL, right?
As I already said, I don't have a strong opinion on which
approach is better. My opinion for the (important) second
point is no. I feel that the pros of a. isn't realistic. If
users want to improve text/csv/binary performance (or
something), they should improve PostgreSQL itself instead of
replacing it as an extension. (Or they should create another
custom copy format such as "faster_text" not "text".)
So I'm OK with the approach b.
>> Are there any missing or wrong items?
>
> I think the approach (b) provides more flexibility than (a) in terms
> of API design as with (a) we need to do everything based on one
> handler function and callbacks.
Thanks for sharing this missing point.
I have a concern that the flexibility may introduce needless
complexity. If it's not a real concern, I'm OK with the
approach b.
>> If we can summarize
>> the current discussion here correctly, others will be able
>> to chime in this discussion. (At least I can do it.)
>
> +1
Are there any more people who are interested in custom COPY
FORMAT implementation design? If no more people, let's
decide it by us.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-12 02:33 ` Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2025-06-12 02:33 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, May 26, 2025 at 10:04:05AM +0900, Sutou Kouhei wrote:
> As I already said, I don't have a strong opinion on which
> approach is better. My opinion for the (important) second
> point is no. I feel that the pros of a. isn't realistic. If
> users want to improve text/csv/binary performance (or
> something), they should improve PostgreSQL itself instead of
> replacing it as an extension. (Or they should create another
> custom copy format such as "faster_text" not "text".)
Patches welcome. Andres may have a TODO board regarding that, I
think.
> So I'm OK with the approach b.
Here is an opinion.
Approach (b), that uses _PG_init() a function to register a custom
format has the merit to be simple to implement and secure by "design",
because it depends only on the fact that we can do a lookup based on
the string defined in one or more DefElems. Adding a dependendy to
search_path as you say could lead to surprising results.
Using a shared ID when a COPY method is registered (like extension
wait events) or an ID that's static in a backend (like EXPLAIN
extensibility does) is an implementation difference that can be useful
for monitoring, and only that AFAIK. If you want to implement
method-based statistics for COPY, you will want to allocate one stats
kind for each COPY method, because the stats stored will be aggregates
of the COPY methods. The stats kind ID is something that should not
be linked to the COPY method ID, because the stats kind ID is
registered in its own dedicated path, and it would be hardcoded in the
library where the COPY callbacks are defined. So you could have a
stats kind with a fixed ID, and a COPY method ID that's linked to each
backend like EXPLAIN does.
One factor to take into account is how much freedom we are OK with
giving to users when it comes to the deployment of custom COPY
methods, and how popular these would be. Cloud is popular these days,
so folks may want to be able to define pointers to functions that are
run in something else than C, as long as the language is trusted. My
take on this part is that we are not going to see many formats out
there that would benefit from these callbacks, so asking for people to
deploy a .so on disk that can only be LOAD'ed or registered with one
of the preloading GUCs should be enough to satisfy most users, even if
the barrier entry to get that only a cloud instead like RDS or Azure
is higher. This has also the benefit in giving more control on the
COPY internals to cloud providers, as they are the ones who would be
in charge of saying if they're OK with a dedicated .so or not. Not
the users themselves. We've had a lot of bad PR and false CVEs in the
past with COPY FROM/TO PROGRAM and the fact that it requires
superusers. Having something in this area that gives more freedom to
the user with something like approach (a) (SQL functions allowed to
define the callback) will, I suspect, bite us back hard.
So, my opinion is to rely on _PG_init(), with a shared ID if you want
to expose the method used somewhere for monitoring tools. You could
as well implement the simpler set of APIs that allocates IDs local to
each backend, like EXPLAIN, then consider later if shared IDs are
really needed. The registration APIs don't have to be fixed in time
across releases, they can be always improved in steps as required.
What matters is ABI compatibility in the same major version once it is
released.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2025-06-12 17:00 ` Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-06-12 17:00 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jun 11, 2025 at 7:34 PM Michael Paquier <[email protected]> wrote:
>
> On Mon, May 26, 2025 at 10:04:05AM +0900, Sutou Kouhei wrote:
> > As I already said, I don't have a strong opinion on which
> > approach is better. My opinion for the (important) second
> > point is no. I feel that the pros of a. isn't realistic. If
> > users want to improve text/csv/binary performance (or
> > something), they should improve PostgreSQL itself instead of
> > replacing it as an extension. (Or they should create another
> > custom copy format such as "faster_text" not "text".)
>
> Patches welcome. Andres may have a TODO board regarding that, I
> think.
>
> > So I'm OK with the approach b.
>
> Here is an opinion.
Thank you for the comments.
>
> Approach (b), that uses _PG_init() a function to register a custom
> format has the merit to be simple to implement and secure by "design",
> because it depends only on the fact that we can do a lookup based on
> the string defined in one or more DefElems. Adding a dependendy to
> search_path as you say could lead to surprising results.
>
> Using a shared ID when a COPY method is registered (like extension
> wait events) or an ID that's static in a backend (like EXPLAIN
> extensibility does) is an implementation difference that can be useful
> for monitoring, and only that AFAIK. If you want to implement
> method-based statistics for COPY, you will want to allocate one stats
> kind for each COPY method, because the stats stored will be aggregates
> of the COPY methods. The stats kind ID is something that should not
> be linked to the COPY method ID, because the stats kind ID is
> registered in its own dedicated path, and it would be hardcoded in the
> library where the COPY callbacks are defined. So you could have a
> stats kind with a fixed ID, and a COPY method ID that's linked to each
> backend like EXPLAIN does.
Good point.
>
> One factor to take into account is how much freedom we are OK with
> giving to users when it comes to the deployment of custom COPY
> methods, and how popular these would be. Cloud is popular these days,
> so folks may want to be able to define pointers to functions that are
> run in something else than C, as long as the language is trusted. My
> take on this part is that we are not going to see many formats out
> there that would benefit from these callbacks, so asking for people to
> deploy a .so on disk that can only be LOAD'ed or registered with one
> of the preloading GUCs should be enough to satisfy most users, even if
> the barrier entry to get that only a cloud instead like RDS or Azure
> is higher. This has also the benefit in giving more control on the
> COPY internals to cloud providers, as they are the ones who would be
> in charge of saying if they're OK with a dedicated .so or not. Not
> the users themselves. We've had a lot of bad PR and false CVEs in the
> past with COPY FROM/TO PROGRAM and the fact that it requires
> superusers. Having something in this area that gives more freedom to
> the user with something like approach (a) (SQL functions allowed to
> define the callback) will, I suspect, bite us back hard.
That's a valid point and I agree.
>
> So, my opinion is to rely on _PG_init(), with a shared ID if you want
> to expose the method used somewhere for monitoring tools. You could
> as well implement the simpler set of APIs that allocates IDs local to
> each backend, like EXPLAIN, then consider later if shared IDs are
> really needed. The registration APIs don't have to be fixed in time
> across releases, they can be always improved in steps as required.
> What matters is ABI compatibility in the same major version once it is
> released.
+1 to start with a simpler set of APIs.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-06-16 23:50 ` Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-06-16 23:50 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBwxgfkMYxgPWyrLG-r8-ptVKjd=jhncY_QAaVJYhQQdw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 12 Jun 2025 10:00:12 -0700,
Masahiko Sawada <[email protected]> wrote:
>> So, my opinion is to rely on _PG_init(), with a shared ID if you want
>> to expose the method used somewhere for monitoring tools. You could
>> as well implement the simpler set of APIs that allocates IDs local to
>> each backend, like EXPLAIN, then consider later if shared IDs are
>> really needed. The registration APIs don't have to be fixed in time
>> across releases, they can be always improved in steps as required.
>> What matters is ABI compatibility in the same major version once it is
>> released.
>
> +1 to start with a simpler set of APIs.
OK. I'll implement the initial version with this
design. (Allocating IDs local not shared.)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-17 00:38 ` Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Michael Paquier @ 2025-06-17 00:38 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote:
> OK. I'll implement the initial version with this
> design. (Allocating IDs local not shared.)
Sounds good to me. Thanks Sutou-san!
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2025-06-18 03:59 ` Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-06-18 03:59 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 17 Jun 2025 09:38:54 +0900,
Michael Paquier <[email protected]> wrote:
> On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote:
>> OK. I'll implement the initial version with this
>> design. (Allocating IDs local not shared.)
>
> Sounds good to me. Thanks Sutou-san!
I've attached the v41 patch set that uses the C API approach
with local (not shared) COPY routine management.
0001: This is same as 0001 in the v40 patch set. It just
cleans up CopySource and CopyDest enums.
0002: This is the initial version of this approach.
Here are some discussion points:
1. This provides 2 registration APIs
(RegisterCopy{From,To}Routine(name, routine)) instead of
1 registration API (RegisterCopyFormat(name,
from_routine, to_routine)).
It's for simple implementation and easy to extend without
breaking APIs in the future. (And some formats may
provide only FROM routine or TO routine.)
Is this design acceptable?
FYI: RegisterCopy{From,To}Routine() uses the same logic
as RegisterExtensionExplainOption().
2. This allocates IDs internally but doesn't provide APIs
that get them. Because it's not needed for now.
We can provide GetExplainExtensionId() like API when we
need it.
Is this design acceptable?
3. I want to register the built-in COPY {FROM,TO} routines
in the PostgreSQL initialization phase. Where should we
do it? In 0002, it's done in InitPostgres() but I'm not
sure whether it's a suitable location or not.
4. 0002 adds CopyFormatOptions::routine as union:
@@ -87,9 +91,14 @@ typedef struct CopyFormatOptions
CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */
int64 reject_limit; /* maximum tolerable number of errors */
List *convert_select; /* list of column names (can be NIL) */
+ union
+ {
+ const struct CopyFromRoutine *from; /* for COPY FROM */
+ const struct CopyToRoutine *to; /* for COPY TO */
+ } routine; /* routine to process the specified format */
} CopyFormatOptions;
Because one of Copy{From,To}Routine is only needed at
once. Is this union usage strange in PostgreSQL?
5. 0002 adds InitializeCopy{From,To}Routines() and
GetCopy{From,To}Routine() that aren't used by COPY
{FROM,TO} routine implementations to copyapi.h. Should we
move them to other .h? If so, which .h should be used for
them?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-24 02:59 ` Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-06-24 02:59 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jun 18, 2025 at 12:59 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 17 Jun 2025 09:38:54 +0900,
> Michael Paquier <[email protected]> wrote:
>
> > On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote:
> >> OK. I'll implement the initial version with this
> >> design. (Allocating IDs local not shared.)
> >
> > Sounds good to me. Thanks Sutou-san!
>
> I've attached the v41 patch set that uses the C API approach
> with local (not shared) COPY routine management.
>
> 0001: This is same as 0001 in the v40 patch set. It just
> cleans up CopySource and CopyDest enums.
> 0002: This is the initial version of this approach.
Thank you for updating the patches!
> Here are some discussion points:
>
> 1. This provides 2 registration APIs
> (RegisterCopy{From,To}Routine(name, routine)) instead of
> 1 registration API (RegisterCopyFormat(name,
> from_routine, to_routine)).
>
> It's for simple implementation and easy to extend without
> breaking APIs in the future. (And some formats may
> provide only FROM routine or TO routine.)
>
> Is this design acceptable?
With the single registration API idea, we can register the custom
format routine that supports only FROM routine using the API like:
RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL);
Compared to this approach, what points do you think having separate
two registration APIs is preferable in terms of extendability and API
compatibility? I think it would be rather confusing for example if
each COPY TO routine and COPY FROM routine is registered by different
extensions with the same format name.
> FYI: RegisterCopy{From,To}Routine() uses the same logic
> as RegisterExtensionExplainOption().
I'm concerned that the patch has duplicated logics for the
registration of COPY FROM and COPY TO.
>
> 2. This allocates IDs internally but doesn't provide APIs
> that get them. Because it's not needed for now.
>
> We can provide GetExplainExtensionId() like API when we
> need it.
>
> Is this design acceptable?
+1
>
> 3. I want to register the built-in COPY {FROM,TO} routines
> in the PostgreSQL initialization phase. Where should we
> do it? In 0002, it's done in InitPostgres() but I'm not
> sure whether it's a suitable location or not.
InitPostgres() is not a correct function as it's a process
initialization function. Probably we don't necessarily need to
register the built-in formats in the same way as custom formats. A
simple solution would be to have separate arrays for built-in formats
and custom formats and have the GetCopy[To|From]Routine() search both
arrays (built-in array first).
> 4. 0002 adds CopyFormatOptions::routine as union:
>
> @@ -87,9 +91,14 @@ typedef struct CopyFormatOptions
> CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */
> int64 reject_limit; /* maximum tolerable number of errors */
> List *convert_select; /* list of column names (can be NIL) */
> + union
> + {
> + const struct CopyFromRoutine *from; /* for COPY FROM */
> + const struct CopyToRoutine *to; /* for COPY TO */
> + } routine; /* routine to process the specified format */
> } CopyFormatOptions;
>
> Because one of Copy{From,To}Routine is only needed at
> once. Is this union usage strange in PostgreSQL?
I think we can live with having two fields as there are other options
that are used only in either COPY FROM or COPY TO.
>
> 5. 0002 adds InitializeCopy{From,To}Routines() and
> GetCopy{From,To}Routine() that aren't used by COPY
> {FROM,TO} routine implementations to copyapi.h. Should we
> move them to other .h? If so, which .h should be used for
> them?
As I commented at 3, I think it's better to avoid dynamically
registering the built-in formats.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-06-24 05:11 ` Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-06-24 05:11 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoA57owo6qYTPTxOtCjDmcuj1tGL1aGs95cvEnoLQvwF0A@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 11:59:17 +0900,
Masahiko Sawada <[email protected]> wrote:
>> 1. This provides 2 registration APIs
>> (RegisterCopy{From,To}Routine(name, routine)) instead of
>> 1 registration API (RegisterCopyFormat(name,
>> from_routine, to_routine)).
>>
>> It's for simple implementation and easy to extend without
>> breaking APIs in the future. (And some formats may
>> provide only FROM routine or TO routine.)
>>
>> Is this design acceptable?
>
> With the single registration API idea, we can register the custom
> format routine that supports only FROM routine using the API like:
>
> RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL);
>
> Compared to this approach, what points do you think having separate
> two registration APIs is preferable in terms of extendability and API
> compatibility?
It's natural to add more related APIs with this
approach. The single registration API provides one feature
by one operation. If we use the RegisterCopyRoutine() for
FROM and TO formats API, it's not natural that we add more
related APIs. In this case, some APIs may provide multiple
features by one operation and other APIs may provide single
feature by one operation. Developers may be confused with
the API. For example, developers may think "what does mean
NULL here?" or "can we use NULL here?" for
"RegisterCopyRoutine("new-format", NewFormatFromRoutine,
NULL)".
> I think it would be rather confusing for example if
> each COPY TO routine and COPY FROM routine is registered by different
> extensions with the same format name.
Hmm, I don't think so. Who is confused by the case? DBA?
Users who use COPY? Why is it confused?
Do you assume the case that the same name is used for
different format? For example, "json" is used for JSON Lines
for COPY FROM and and normal JSON for COPY TO by different
extensions.
>> FYI: RegisterCopy{From,To}Routine() uses the same logic
>> as RegisterExtensionExplainOption().
>
> I'm concerned that the patch has duplicated logics for the
> registration of COPY FROM and COPY TO.
We can implement a convenient routine that can be used for
RegisterExtensionExplainOption() and
RegisterCopy{From,To}Routine() if it's needed.
>> 3. I want to register the built-in COPY {FROM,TO} routines
>> in the PostgreSQL initialization phase. Where should we
>> do it? In 0002, it's done in InitPostgres() but I'm not
>> sure whether it's a suitable location or not.
>
> InitPostgres() is not a correct function as it's a process
> initialization function. Probably we don't necessarily need to
> register the built-in formats in the same way as custom formats. A
> simple solution would be to have separate arrays for built-in formats
> and custom formats and have the GetCopy[To|From]Routine() search both
> arrays (built-in array first).
We had a discussion that we should dog-food APIs:
https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4...
> We should (and usually do) dog-food APIs when reasonable
> and this situation seems quite reasonable.
In this case, we don't need to dog-food APIs, right?
>> 4. 0002 adds CopyFormatOptions::routine as union:
>>
>> @@ -87,9 +91,14 @@ typedef struct CopyFormatOptions
>> CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */
>> int64 reject_limit; /* maximum tolerable number of errors */
>> List *convert_select; /* list of column names (can be NIL) */
>> + union
>> + {
>> + const struct CopyFromRoutine *from; /* for COPY FROM */
>> + const struct CopyToRoutine *to; /* for COPY TO */
>> + } routine; /* routine to process the specified format */
>> } CopyFormatOptions;
>>
>> Because one of Copy{From,To}Routine is only needed at
>> once. Is this union usage strange in PostgreSQL?
>
> I think we can live with having two fields as there are other options
> that are used only in either COPY FROM or COPY TO.
OK.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-24 06:24 ` Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-06-24 06:24 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jun 24, 2025 at 2:11 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoA57owo6qYTPTxOtCjDmcuj1tGL1aGs95cvEnoLQvwF0A@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 11:59:17 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
> >> 1. This provides 2 registration APIs
> >> (RegisterCopy{From,To}Routine(name, routine)) instead of
> >> 1 registration API (RegisterCopyFormat(name,
> >> from_routine, to_routine)).
> >>
> >> It's for simple implementation and easy to extend without
> >> breaking APIs in the future. (And some formats may
> >> provide only FROM routine or TO routine.)
> >>
> >> Is this design acceptable?
> >
> > With the single registration API idea, we can register the custom
> > format routine that supports only FROM routine using the API like:
> >
> > RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL);
> >
> > Compared to this approach, what points do you think having separate
> > two registration APIs is preferable in terms of extendability and API
> > compatibility?
>
> It's natural to add more related APIs with this
> approach. The single registration API provides one feature
> by one operation. If we use the RegisterCopyRoutine() for
> FROM and TO formats API, it's not natural that we add more
> related APIs. In this case, some APIs may provide multiple
> features by one operation and other APIs may provide single
> feature by one operation. Developers may be confused with
> the API. For example, developers may think "what does mean
> NULL here?" or "can we use NULL here?" for
> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
> NULL)".
We can document it in the comment for the registration function.
>
>
> > I think it would be rather confusing for example if
> > each COPY TO routine and COPY FROM routine is registered by different
> > extensions with the same format name.
>
> Hmm, I don't think so. Who is confused by the case? DBA?
> Users who use COPY? Why is it confused?
>
> Do you assume the case that the same name is used for
> different format? For example, "json" is used for JSON Lines
> for COPY FROM and and normal JSON for COPY TO by different
> extensions.
Suppose that extension-A implements only CopyToRoutine for the
custom-format-X with the format name 'myformat' and extension-B
implements only CopyFromRoutine for the custom-format-Y with the same
name, if users load both extension-A and extension-B, it seems to me
that extension-A registers the custom-format-X format as 'myformat'
only with CopyToRoutine, and extension-B overwrites the 'myformat'
registration by adding custom-format-Y's CopyFromRoutine. However, if
users register extension-C that implements both routines with the
format name 'myformat', they can register neither extension-A nor
extension-B, which seems to me that we don't allow overwriting the
registration in this case.
I think the core issue appears to be the internal management of custom
format entries but the current patch does enable registration
overwriting in the former case (extension-A and extension-B case).
>
> >> FYI: RegisterCopy{From,To}Routine() uses the same logic
> >> as RegisterExtensionExplainOption().
> >
> > I'm concerned that the patch has duplicated logics for the
> > registration of COPY FROM and COPY TO.
>
> We can implement a convenient routine that can be used for
> RegisterExtensionExplainOption() and
> RegisterCopy{From,To}Routine() if it's needed.
I meant there are duplicated codes in COPY FROM and COPY TO. For
instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have
the same logic.
>
> >> 3. I want to register the built-in COPY {FROM,TO} routines
> >> in the PostgreSQL initialization phase. Where should we
> >> do it? In 0002, it's done in InitPostgres() but I'm not
> >> sure whether it's a suitable location or not.
> >
> > InitPostgres() is not a correct function as it's a process
> > initialization function. Probably we don't necessarily need to
> > register the built-in formats in the same way as custom formats. A
> > simple solution would be to have separate arrays for built-in formats
> > and custom formats and have the GetCopy[To|From]Routine() search both
> > arrays (built-in array first).
>
> We had a discussion that we should dog-food APIs:
>
> https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4...
>
> > We should (and usually do) dog-food APIs when reasonable
> > and this situation seems quite reasonable.
>
> In this case, we don't need to dog-food APIs, right?
Yes, I think so.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-06-24 07:10 ` Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-06-24 07:10 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoC8-d=GF-hOvGqUyq2xFg=QGpYfCiWJbcp4wcn0UidrPw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 15:24:23 +0900,
Masahiko Sawada <[email protected]> wrote:
>> It's natural to add more related APIs with this
>> approach. The single registration API provides one feature
>> by one operation. If we use the RegisterCopyRoutine() for
>> FROM and TO formats API, it's not natural that we add more
>> related APIs. In this case, some APIs may provide multiple
>> features by one operation and other APIs may provide single
>> feature by one operation. Developers may be confused with
>> the API. For example, developers may think "what does mean
>> NULL here?" or "can we use NULL here?" for
>> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
>> NULL)".
>
> We can document it in the comment for the registration function.
I think that API that can be understandable without the
additional note is better API than API that needs some
notes.
Why do you suggest the RegisterCopyRoutine("new-format",
NewFormatFromRoutine, NewFormatToRoutine) API? You want to
remove the duplicated codes in
RegisterCopy{From,To}Routine(), right? I think that we can
do it by creating a convenient function that has the
duplicated codes extracted from
RegisterCopy{From,To}Routine() and
RegisterExtensionExplainOption().
BTW, what do you think about my answer (one feature by one
operation API is more extendable API) for your question
(extendability and API compatibility)?
> Suppose that extension-A implements only CopyToRoutine for the
> custom-format-X with the format name 'myformat' and extension-B
> implements only CopyFromRoutine for the custom-format-Y with the same
> name, if users load both extension-A and extension-B, it seems to me
> that extension-A registers the custom-format-X format as 'myformat'
> only with CopyToRoutine, and extension-B overwrites the 'myformat'
> registration by adding custom-format-Y's CopyFromRoutine. However, if
> users register extension-C that implements both routines with the
> format name 'myformat', they can register neither extension-A nor
> extension-B, which seems to me that we don't allow overwriting the
> registration in this case.
Do you assume that users use extension-A, extension-B and
extension-C without reading their documentation? If users
read their documentation before users use them, users can
know all of them use the same format name 'myformat' and
which extension provides Copy{From,To}Routine.
In this case, these users (who don't read documentation)
will be confused with the RegisterCopyRoutine("new-format",
NewFormatFromRoutine, NewFormatToRoutine) API too. Do we
really need to care about this case?
> I think the core issue appears to be the internal management of custom
> format entries but the current patch does enable registration
> overwriting in the former case (extension-A and extension-B case).
This is the same behavior as existing custom EXPLAIN option
implementation. Should we use different behavior here?
>> >> FYI: RegisterCopy{From,To}Routine() uses the same logic
>> >> as RegisterExtensionExplainOption().
>> >
>> > I'm concerned that the patch has duplicated logics for the
>> > registration of COPY FROM and COPY TO.
>>
>> We can implement a convenient routine that can be used for
>> RegisterExtensionExplainOption() and
>> RegisterCopy{From,To}Routine() if it's needed.
>
> I meant there are duplicated codes in COPY FROM and COPY TO. For
> instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have
> the same logic.
Yes, I understand it. I wanted to say that we can remove the
duplicated codes by introducing a RegisterSomething()
function that can be used by
RegisterExtensionExplainOption() and
RegisterCopy{From,To}Routine():
void
RegisterSomething(...)
{
/* Common codes in RegisterExtensionExplainOption() and
RegisterCopy{From,To}Routine()
...
*/
}
void
RegisterExtensionExplainOption(...)
{
RegisterSomething(...);
}
void
RegisterCopyFromRoutine(...)
{
RegisterSomething(...);
}
void
RegisterCopyToRoutine(...)
{
RegisterSomething(...);
}
You think that this approach can't remove the duplicated
codes, right?
>> > InitPostgres() is not a correct function as it's a process
>> > initialization function. Probably we don't necessarily need to
>> > register the built-in formats in the same way as custom formats. A
>> > simple solution would be to have separate arrays for built-in formats
>> > and custom formats and have the GetCopy[To|From]Routine() search both
>> > arrays (built-in array first).
>>
>> We had a discussion that we should dog-food APIs:
>>
>> https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4...
>>
>> > We should (and usually do) dog-food APIs when reasonable
>> > and this situation seems quite reasonable.
>>
>> In this case, we don't need to dog-food APIs, right?
>
> Yes, I think so.
OK. I don't have a strong opinion for it. If nobody objects
it, I'll do it when I update the patch set.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-24 15:48 ` Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-06-24 15:48 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jun 24, 2025 at 4:10 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoC8-d=GF-hOvGqUyq2xFg=QGpYfCiWJbcp4wcn0UidrPw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 15:24:23 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
> >> It's natural to add more related APIs with this
> >> approach. The single registration API provides one feature
> >> by one operation. If we use the RegisterCopyRoutine() for
> >> FROM and TO formats API, it's not natural that we add more
> >> related APIs. In this case, some APIs may provide multiple
> >> features by one operation and other APIs may provide single
> >> feature by one operation. Developers may be confused with
> >> the API. For example, developers may think "what does mean
> >> NULL here?" or "can we use NULL here?" for
> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
> >> NULL)".
> >
> > We can document it in the comment for the registration function.
>
> I think that API that can be understandable without the
> additional note is better API than API that needs some
> notes.
I don't see much difference in this case.
>
> Why do you suggest the RegisterCopyRoutine("new-format",
> NewFormatFromRoutine, NewFormatToRoutine) API? You want to
> remove the duplicated codes in
> RegisterCopy{From,To}Routine(), right?
No. I think that if extensions are likely to support both
CopyToRoutine and CopyFromRoutine in most cases, it would be simpler
to register the custom format using a single API. Registering
CopyToRoutine and CopyFromRoutine separately seems redundant to me.
>
> BTW, what do you think about my answer (one feature by one
> operation API is more extendable API) for your question
> (extendability and API compatibility)?
Could you provide some examples? It seems to me that even if we
provide the single API for the registration we can provide other APIs
differently. For example, if we want to provide an API to register a
custom option, we can provide RegisterCopyToOption() and
RegisterCopyFromOption().
>
> > Suppose that extension-A implements only CopyToRoutine for the
> > custom-format-X with the format name 'myformat' and extension-B
> > implements only CopyFromRoutine for the custom-format-Y with the same
> > name, if users load both extension-A and extension-B, it seems to me
> > that extension-A registers the custom-format-X format as 'myformat'
> > only with CopyToRoutine, and extension-B overwrites the 'myformat'
> > registration by adding custom-format-Y's CopyFromRoutine. However, if
> > users register extension-C that implements both routines with the
> > format name 'myformat', they can register neither extension-A nor
> > extension-B, which seems to me that we don't allow overwriting the
> > registration in this case.
>
> Do you assume that users use extension-A, extension-B and
> extension-C without reading their documentation? If users
> read their documentation before users use them, users can
> know all of them use the same format name 'myformat' and
> which extension provides Copy{From,To}Routine.
>
> In this case, these users (who don't read documentation)
> will be confused with the RegisterCopyRoutine("new-format",
> NewFormatFromRoutine, NewFormatToRoutine) API too. Do we
> really need to care about this case?
My point is about the consistency of registration behavior. I think
that we should raise an error if the custom format name that an
extension tries to register already exists. Therefore I'm not sure why
installing extension-A+B is okay but installing extension-C+A or
extension-C+B is not okay? We can think that's an extension-A's choice
not to implement CopyFromRoutine for the 'myformat' format so
extension-B should not change it.
>
> > I think the core issue appears to be the internal management of custom
> > format entries but the current patch does enable registration
> > overwriting in the former case (extension-A and extension-B case).
>
> This is the same behavior as existing custom EXPLAIN option
> implementation. Should we use different behavior here?
I think that unlike custom EXPLAIN options, it's better to raise an
error or a warning if the custom format name (or combination of format
name and COPY direction) that an extension tries to register already
exists.
> >> >> FYI: RegisterCopy{From,To}Routine() uses the same logic
> >> >> as RegisterExtensionExplainOption().
> >> >
> >> > I'm concerned that the patch has duplicated logics for the
> >> > registration of COPY FROM and COPY TO.
> >>
> >> We can implement a convenient routine that can be used for
> >> RegisterExtensionExplainOption() and
> >> RegisterCopy{From,To}Routine() if it's needed.
> >
> > I meant there are duplicated codes in COPY FROM and COPY TO. For
> > instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have
> > the same logic.
>
> Yes, I understand it. I wanted to say that we can remove the
> duplicated codes by introducing a RegisterSomething()
> function that can be used by
> RegisterExtensionExplainOption() and
> RegisterCopy{From,To}Routine():
>
> void
> RegisterSomething(...)
> {
> /* Common codes in RegisterExtensionExplainOption() and
> RegisterCopy{From,To}Routine()
> ...
> */
> }
>
> void
> RegisterExtensionExplainOption(...)
> {
> RegisterSomething(...);
> }
>
> void
> RegisterCopyFromRoutine(...)
> {
> RegisterSomething(...);
> }
>
> void
> RegisterCopyToRoutine(...)
> {
> RegisterSomething(...);
> }
>
> You think that this approach can't remove the duplicated
> codes, right?
Well, no, I just meant we don't need to do that. Custom EXPLAIN option
and custom COPY format are different features and have different
requirements. I think while we don't need to remove duplicates between
them at least at this stage we need to remove the duplicate between
COPY TO registration code and COPY TO's one.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-06-25 07:35 ` Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-06-25 07:35 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900,
Masahiko Sawada <[email protected]> wrote:
>> >> It's natural to add more related APIs with this
>> >> approach. The single registration API provides one feature
>> >> by one operation. If we use the RegisterCopyRoutine() for
>> >> FROM and TO formats API, it's not natural that we add more
>> >> related APIs. In this case, some APIs may provide multiple
>> >> features by one operation and other APIs may provide single
>> >> feature by one operation. Developers may be confused with
>> >> the API. For example, developers may think "what does mean
>> >> NULL here?" or "can we use NULL here?" for
>> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
>> >> NULL)".
>> >
>> > We can document it in the comment for the registration function.
>>
>> I think that API that can be understandable without the
>> additional note is better API than API that needs some
>> notes.
>
> I don't see much difference in this case.
OK. It seems that we can't agree on which API is better.
I've implemented your idea as the v42 patch set. Can we
proceed this proposal with this approach? What is the next
step?
> No. I think that if extensions are likely to support both
> CopyToRoutine and CopyFromRoutine in most cases, it would be simpler
> to register the custom format using a single API. Registering
> CopyToRoutine and CopyFromRoutine separately seems redundant to me.
I don't think so. In general, extensions are implemented
step by step. Extension developers will not implement
CopyToRoutine and CopyFromRoutine at once even if extensions
implement both of CopyToRoutine and CopyFromRoutine
eventually.
> Could you provide some examples? It seems to me that even if we
> provide the single API for the registration we can provide other APIs
> differently. For example, if we want to provide an API to register a
> custom option, we can provide RegisterCopyToOption() and
> RegisterCopyFromOption().
Yes. We can mix different style APIs. In general, consistent
style APIs is easier to use than mixed style APIs. If it's
not an important point in PostgreSQL API design, my point is
meaningless. (Sorry, I'm not familiar with PostgreSQL API
design.)
> My point is about the consistency of registration behavior. I think
> that we should raise an error if the custom format name that an
> extension tries to register already exists. Therefore I'm not sure why
> installing extension-A+B is okay but installing extension-C+A or
> extension-C+B is not okay? We can think that's an extension-A's choice
> not to implement CopyFromRoutine for the 'myformat' format so
> extension-B should not change it.
I think that it's the users' responsibility. I think that
it's more convenient that users can mix extension-A+B (A
provides only TO format and B provides only FROM format)
than users can't mix them. I think that extension-A doesn't
want to prohibit FROM format in the case. Extension-A just
doesn't care about FROM format.
FYI: Both of extension-C+A and extension-C+B are OK when we
update not raising an error existing format.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-06-30 06:00 ` Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-06-30 06:00 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jun 25, 2025 at 4:35 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
> >> >> It's natural to add more related APIs with this
> >> >> approach. The single registration API provides one feature
> >> >> by one operation. If we use the RegisterCopyRoutine() for
> >> >> FROM and TO formats API, it's not natural that we add more
> >> >> related APIs. In this case, some APIs may provide multiple
> >> >> features by one operation and other APIs may provide single
> >> >> feature by one operation. Developers may be confused with
> >> >> the API. For example, developers may think "what does mean
> >> >> NULL here?" or "can we use NULL here?" for
> >> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
> >> >> NULL)".
> >> >
> >> > We can document it in the comment for the registration function.
> >>
> >> I think that API that can be understandable without the
> >> additional note is better API than API that needs some
> >> notes.
> >
> > I don't see much difference in this case.
>
> OK. It seems that we can't agree on which API is better.
>
> I've implemented your idea as the v42 patch set. Can we
> proceed this proposal with this approach? What is the next
> step?
I'll review the patches. In the meanwhile could you update the
documentation accordingly?
>
> > No. I think that if extensions are likely to support both
> > CopyToRoutine and CopyFromRoutine in most cases, it would be simpler
> > to register the custom format using a single API. Registering
> > CopyToRoutine and CopyFromRoutine separately seems redundant to me.
>
> I don't think so. In general, extensions are implemented
> step by step. Extension developers will not implement
> CopyToRoutine and CopyFromRoutine at once even if extensions
> implement both of CopyToRoutine and CopyFromRoutine
> eventually.
Hmm, I think if the extension eventually implements both directions,
it would make sense to provide the single API.
>
> > Could you provide some examples? It seems to me that even if we
> > provide the single API for the registration we can provide other APIs
> > differently. For example, if we want to provide an API to register a
> > custom option, we can provide RegisterCopyToOption() and
> > RegisterCopyFromOption().
>
> Yes. We can mix different style APIs. In general, consistent
> style APIs is easier to use than mixed style APIs. If it's
> not an important point in PostgreSQL API design, my point is
> meaningless. (Sorry, I'm not familiar with PostgreSQL API
> design.)
As far as I know, there is no standard for PostgreSQL API design, but
I don't find any weirdness in this design.
>
> > My point is about the consistency of registration behavior. I think
> > that we should raise an error if the custom format name that an
> > extension tries to register already exists. Therefore I'm not sure why
> > installing extension-A+B is okay but installing extension-C+A or
> > extension-C+B is not okay? We can think that's an extension-A's choice
> > not to implement CopyFromRoutine for the 'myformat' format so
> > extension-B should not change it.
>
> I think that it's the users' responsibility. I think that
> it's more convenient that users can mix extension-A+B (A
> provides only TO format and B provides only FROM format)
> than users can't mix them. I think that extension-A doesn't
> want to prohibit FROM format in the case. Extension-A just
> doesn't care about FROM format.
>
> FYI: Both of extension-C+A and extension-C+B are OK when we
> update not raising an error existing format.
I want to keep the basic design that one custom format comes from one
extension because it's straightforward for both of us and users and
easy to maintain format ID. IIUC we somewhat agreed on this design in
the previous API design (TABLESAMPLE like API).
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-13 18:28 ` Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 12:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andres Freund <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Masahiko Sawada @ 2025-07-13 18:28 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jun 30, 2025 at 3:00 PM Masahiko Sawada <[email protected]> wrote:
>
> On Wed, Jun 25, 2025 at 4:35 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900,
> > Masahiko Sawada <[email protected]> wrote:
> >
> > >> >> It's natural to add more related APIs with this
> > >> >> approach. The single registration API provides one feature
> > >> >> by one operation. If we use the RegisterCopyRoutine() for
> > >> >> FROM and TO formats API, it's not natural that we add more
> > >> >> related APIs. In this case, some APIs may provide multiple
> > >> >> features by one operation and other APIs may provide single
> > >> >> feature by one operation. Developers may be confused with
> > >> >> the API. For example, developers may think "what does mean
> > >> >> NULL here?" or "can we use NULL here?" for
> > >> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine,
> > >> >> NULL)".
> > >> >
> > >> > We can document it in the comment for the registration function.
> > >>
> > >> I think that API that can be understandable without the
> > >> additional note is better API than API that needs some
> > >> notes.
> > >
> > > I don't see much difference in this case.
> >
> > OK. It seems that we can't agree on which API is better.
> >
> > I've implemented your idea as the v42 patch set. Can we
> > proceed this proposal with this approach? What is the next
> > step?
>
> I'll review the patches.
I've reviewed the 0001 and 0002 patches. The API implemented in the
0002 patch looks good to me, but I'm concerned about the capsulation
of copy state data. With the v42 patches, we pass the whole
CopyToStateData to the extension codes, but most of the fields in
CopyToStateData are internal working state data that shouldn't be
exposed to extensions. I think we need to sort out which fields are
exposed or not. That way, it would be safer and we would be able to
avoid exposing copyto_internal.h and extensions would not need to
include copyfrom_internal.h.
I've implemented a draft patch for that idea. In the 0001 patch, I
moved fields that are related to internal working state from
CopyToStateData to CopyToExectuionData. COPY routine APIs pass a
pointer of CopyToStateData but extensions can access only fields
except for CopyToExectuionData. In the 0002 patch, I've implemented
the registration API and some related APIs based on your v42 patch.
I've made similar changes to COPY FROM codes too.
The patch is a very PoC phase and we would need to scrutinize the
fields that should or should not be exposed. Feedback is very welcome.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] 0001-Refactor-CopyToStateData-and-CopyFromStateData.patch (80.9K, ../../CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com/2-0001-Refactor-CopyToStateData-and-CopyFromStateData.patch)
download | inline diff:
From f80e7ea77a4faa2abd64782e9bdfc20ad8d0eff3 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Sun, 13 Jul 2025 06:50:40 -0700
Subject: [PATCH 1/2] Refactor CopyToStateData and CopyFromStateData.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
src/backend/commands/copyfrom.c | 238 ++++++------
src/backend/commands/copyfromparse.c | 446 ++++++++++++-----------
src/backend/commands/copyto.c | 111 +++---
src/include/commands/copy.h | 55 ++-
src/include/commands/copyfrom_internal.h | 22 +-
5 files changed, 457 insertions(+), 415 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index fbbbc09a97b..03779838654 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -168,6 +168,7 @@ CopyFromGetRoutine(const CopyFormatOptions *opts)
static void
CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc)
{
+ CopyFromExecutionData *edata = cstate->edata;
AttrNumber attr_count;
/*
@@ -175,24 +176,24 @@ CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc)
* converted input data. Otherwise, we can just point input_buf to the
* same buffer as raw_buf.
*/
- if (cstate->need_transcoding)
+ if (cstate->edata->need_transcoding)
{
- cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
- cstate->input_buf_index = cstate->input_buf_len = 0;
+ edata->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
+ edata->input_buf_index = edata->input_buf_len = 0;
}
else
- cstate->input_buf = cstate->raw_buf;
- cstate->input_reached_eof = false;
+ edata->input_buf = edata->raw_buf;
+ edata->input_reached_eof = false;
- initStringInfo(&cstate->line_buf);
+ initStringInfo(&edata->line_buf);
/*
* Create workspace for CopyReadAttributes results; used by CSV and text
* format.
*/
attr_count = list_length(cstate->attnumlist);
- cstate->max_fields = attr_count;
- cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
+ edata->max_fields = attr_count;
+ edata->raw_fields = (char **) palloc(attr_count * sizeof(char *));
}
/*
@@ -254,48 +255,49 @@ void
CopyFromErrorCallback(void *arg)
{
CopyFromState cstate = (CopyFromState) arg;
+ CopyFromExecutionData *edata = cstate->edata;
- if (cstate->relname_only)
+ if (edata->relname_only)
{
errcontext("COPY %s",
- cstate->cur_relname);
+ edata->cur_relname);
return;
}
if (cstate->opts.binary)
{
/* can't usefully display the data */
- if (cstate->cur_attname)
+ if (edata->cur_attname)
errcontext("COPY %s, line %" PRIu64 ", column %s",
- cstate->cur_relname,
- cstate->cur_lineno,
- cstate->cur_attname);
+ edata->cur_relname,
+ edata->cur_lineno,
+ edata->cur_attname);
else
errcontext("COPY %s, line %" PRIu64,
- cstate->cur_relname,
- cstate->cur_lineno);
+ edata->cur_relname,
+ edata->cur_lineno);
}
else
{
- if (cstate->cur_attname && cstate->cur_attval)
+ if (edata->cur_attname && edata->cur_attval)
{
/* error is relevant to a particular column */
char *attval;
- attval = CopyLimitPrintoutLength(cstate->cur_attval);
+ attval = CopyLimitPrintoutLength(edata->cur_attval);
errcontext("COPY %s, line %" PRIu64 ", column %s: \"%s\"",
- cstate->cur_relname,
- cstate->cur_lineno,
- cstate->cur_attname,
+ edata->cur_relname,
+ edata->cur_lineno,
+ edata->cur_attname,
attval);
pfree(attval);
}
- else if (cstate->cur_attname)
+ else if (edata->cur_attname)
{
/* error is relevant to a particular column, value is NULL */
errcontext("COPY %s, line %" PRIu64 ", column %s: null input",
- cstate->cur_relname,
- cstate->cur_lineno,
- cstate->cur_attname);
+ edata->cur_relname,
+ edata->cur_lineno,
+ edata->cur_attname);
}
else
{
@@ -304,21 +306,21 @@ CopyFromErrorCallback(void *arg)
*
* If line_buf still contains the correct line, print it.
*/
- if (cstate->line_buf_valid)
+ if (edata->line_buf_valid)
{
char *lineval;
- lineval = CopyLimitPrintoutLength(cstate->line_buf.data);
+ lineval = CopyLimitPrintoutLength(edata->line_buf.data);
errcontext("COPY %s, line %" PRIu64 ": \"%s\"",
- cstate->cur_relname,
- cstate->cur_lineno, lineval);
+ edata->cur_relname,
+ edata->cur_lineno, lineval);
pfree(lineval);
}
else
{
errcontext("COPY %s, line %" PRIu64,
- cstate->cur_relname,
- cstate->cur_lineno);
+ edata->cur_relname,
+ edata->cur_lineno);
}
}
}
@@ -448,6 +450,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
int64 *processed)
{
CopyFromState cstate = miinfo->cstate;
+ CopyFromExecutionData *edata = cstate->edata;
EState *estate = miinfo->estate;
int nused = buffer->nused;
ResultRelInfo *resultRelInfo = buffer->resultRelInfo;
@@ -469,8 +472,8 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
* We suppress error context information other than the relation name,
* if one of the operations below fails.
*/
- Assert(!cstate->relname_only);
- cstate->relname_only = true;
+ Assert(!edata->relname_only);
+ edata->relname_only = true;
while (sent < nused)
{
@@ -514,7 +517,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
ExecARInsertTriggers(estate, resultRelInfo,
slot, NIL,
- cstate->transition_capture);
+ edata->transition_capture);
}
}
@@ -528,14 +531,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
ExecClearTuple(slots[i]);
/* reset relname_only */
- cstate->relname_only = false;
+ edata->relname_only = false;
}
else
{
CommandId mycid = miinfo->mycid;
int ti_options = miinfo->ti_options;
- bool line_buf_valid = cstate->line_buf_valid;
- uint64 save_cur_lineno = cstate->cur_lineno;
+ bool line_buf_valid = edata->line_buf_valid;
+ uint64 save_cur_lineno = edata->cur_lineno;
MemoryContext oldcontext;
Assert(buffer->bistate != NULL);
@@ -544,7 +547,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
* Print error context information correctly, if one of the operations
* below fails.
*/
- cstate->line_buf_valid = false;
+ edata->line_buf_valid = false;
/*
* table_multi_insert may leak memory, so switch to short-lived memory
@@ -569,14 +572,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
{
List *recheckIndexes;
- cstate->cur_lineno = buffer->linenos[i];
+ edata->cur_lineno = buffer->linenos[i];
recheckIndexes =
ExecInsertIndexTuples(resultRelInfo,
buffer->slots[i], estate, false,
false, NULL, NIL, false);
ExecARInsertTriggers(estate, resultRelInfo,
slots[i], recheckIndexes,
- cstate->transition_capture);
+ edata->transition_capture);
list_free(recheckIndexes);
}
@@ -588,10 +591,10 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
(resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
resultRelInfo->ri_TrigDesc->trig_insert_new_table))
{
- cstate->cur_lineno = buffer->linenos[i];
+ edata->cur_lineno = buffer->linenos[i];
ExecARInsertTriggers(estate, resultRelInfo,
slots[i], NIL,
- cstate->transition_capture);
+ edata->transition_capture);
}
ExecClearTuple(slots[i]);
@@ -603,8 +606,8 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
*processed);
/* reset cur_lineno and line_buf_valid to what they were */
- cstate->line_buf_valid = line_buf_valid;
- cstate->cur_lineno = save_cur_lineno;
+ edata->line_buf_valid = line_buf_valid;
+ edata->cur_lineno = save_cur_lineno;
}
/* Mark that all slots are free */
@@ -778,6 +781,7 @@ CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
uint64
CopyFrom(CopyFromState cstate)
{
+ CopyFromExecutionData *edata = cstate->edata;
ResultRelInfo *resultRelInfo;
ResultRelInfo *target_resultRelInfo;
ResultRelInfo *prevResultRelInfo = NULL;
@@ -801,10 +805,10 @@ CopyFrom(CopyFromState cstate)
bool leafpart_use_multi_insert = false;
Assert(cstate->rel);
- Assert(list_length(cstate->range_table) == 1);
+ Assert(list_length(edata->range_table) == 1);
if (cstate->opts.on_error != COPY_ON_ERROR_STOP)
- Assert(cstate->escontext);
+ Assert(edata->escontext);
/*
* The target must be a plain, foreign, or partitioned relation, or have
@@ -913,7 +917,7 @@ CopyFrom(CopyFromState cstate)
* index-entry-making machinery. (There used to be a huge amount of code
* here that basically duplicated execUtils.c ...)
*/
- ExecInitRangeTable(estate, cstate->range_table, cstate->rteperminfos,
+ ExecInitRangeTable(estate, edata->range_table, edata->rteperminfos,
bms_make_singleton(1));
resultRelInfo = target_resultRelInfo = makeNode(ResultRelInfo);
ExecInitResultRelation(estate, resultRelInfo, 1);
@@ -968,7 +972,7 @@ CopyFrom(CopyFromState cstate)
* transition capture is active, we also set it in mtstate, which is
* passed to ExecFindPartition() below.
*/
- cstate->transition_capture = mtstate->mt_transition_capture =
+ edata->transition_capture = mtstate->mt_transition_capture =
MakeTransitionCaptureState(cstate->rel->trigdesc,
RelationGetRelid(cstate->rel),
CMD_INSERT);
@@ -980,9 +984,9 @@ CopyFrom(CopyFromState cstate)
if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
proute = ExecSetupPartitionTupleRouting(estate, cstate->rel);
- if (cstate->whereClause)
- cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
- &mtstate->ps);
+ if (edata->whereClause)
+ edata->qualexpr = ExecInitQual(castNode(List, edata->whereClause),
+ &mtstate->ps);
/*
* It's generally more efficient to prepare a bunch of tuples for
@@ -1025,7 +1029,7 @@ CopyFrom(CopyFromState cstate)
*/
insertMethod = CIM_SINGLE;
}
- else if (cstate->volatile_defexprs)
+ else if (edata->volatile_defexprs)
{
/*
* Can't support multi-inserts if there are any volatile default
@@ -1038,7 +1042,7 @@ CopyFrom(CopyFromState cstate)
*/
insertMethod = CIM_SINGLE;
}
- else if (contain_volatile_functions(cstate->whereClause))
+ else if (contain_volatile_functions(edata->whereClause))
{
/*
* Can't support multi-inserts if there are any volatile function
@@ -1150,7 +1154,7 @@ CopyFrom(CopyFromState cstate)
break;
if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE &&
- cstate->escontext->error_occurred)
+ edata->escontext->error_occurred)
{
/*
* Soft error occurred, skip this tuple and just make
@@ -1158,14 +1162,14 @@ CopyFrom(CopyFromState cstate)
* don't set details_wanted and error_data is not to be filled,
* just resetting error_occurred is enough.
*/
- cstate->escontext->error_occurred = false;
+ edata->escontext->error_occurred = false;
/* Report that this tuple was skipped by the ON_ERROR clause */
pgstat_progress_update_param(PROGRESS_COPY_TUPLES_SKIPPED,
- cstate->num_errors);
+ edata->num_errors);
if (cstate->opts.reject_limit > 0 &&
- cstate->num_errors > cstate->opts.reject_limit)
+ edata->num_errors > cstate->opts.reject_limit)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("skipped more than REJECT_LIMIT (%" PRId64 ") rows due to data type incompatibility",
@@ -1186,11 +1190,11 @@ CopyFrom(CopyFromState cstate)
/* Triggers and stuff need to be invoked in query context. */
MemoryContextSwitchTo(oldcontext);
- if (cstate->whereClause)
+ if (edata->whereClause)
{
econtext->ecxt_scantuple = myslot;
/* Skip items that don't match COPY's WHERE clause */
- if (!ExecQual(cstate->qualexpr, econtext))
+ if (!ExecQual(edata->qualexpr, econtext))
{
/*
* Report that this tuple was filtered out by the WHERE
@@ -1266,8 +1270,8 @@ CopyFrom(CopyFromState cstate)
* we can just remember the original unconverted tuple to avoid a
* needless round trip conversion.
*/
- if (cstate->transition_capture != NULL)
- cstate->transition_capture->tcs_original_insert_tuple =
+ if (edata->transition_capture != NULL)
+ edata->transition_capture->tcs_original_insert_tuple =
!has_before_insert_row_trig ? myslot : NULL;
/*
@@ -1379,8 +1383,8 @@ CopyFrom(CopyFromState cstate)
/* Add this tuple to the tuple buffer */
CopyMultiInsertInfoStore(&multiInsertInfo,
resultRelInfo, myslot,
- cstate->line_buf.len,
- cstate->cur_lineno);
+ edata->line_buf.len,
+ edata->cur_lineno);
/*
* If enough inserts have queued up, then flush all
@@ -1440,7 +1444,7 @@ CopyFrom(CopyFromState cstate)
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, myslot,
- recheckIndexes, cstate->transition_capture);
+ recheckIndexes, edata->transition_capture);
list_free(recheckIndexes);
}
@@ -1468,13 +1472,13 @@ CopyFrom(CopyFromState cstate)
error_context_stack = errcallback.previous;
if (cstate->opts.on_error != COPY_ON_ERROR_STOP &&
- cstate->num_errors > 0 &&
+ edata->num_errors > 0 &&
cstate->opts.log_verbosity >= COPY_LOG_VERBOSITY_DEFAULT)
ereport(NOTICE,
errmsg_plural("%" PRIu64 " row was skipped due to data type incompatibility",
"%" PRIu64 " rows were skipped due to data type incompatibility",
- cstate->num_errors,
- cstate->num_errors));
+ edata->num_errors,
+ edata->num_errors));
if (bistate != NULL)
FreeBulkInsertState(bistate);
@@ -1482,7 +1486,7 @@ CopyFrom(CopyFromState cstate)
MemoryContextSwitchTo(oldcontext);
/* Execute AFTER STATEMENT insertion triggers */
- ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
+ ExecASInsertTriggers(estate, target_resultRelInfo, edata->transition_capture);
/* Handle queued AFTER triggers */
AfterTriggerEndQuery(estate);
@@ -1536,6 +1540,7 @@ BeginCopyFrom(ParseState *pstate,
List *options)
{
CopyFromState cstate;
+ CopyFromExecutionData *edata;
bool pipe = (filename == NULL);
TupleDesc tupDesc;
AttrNumber num_phys_attrs,
@@ -1560,15 +1565,18 @@ BeginCopyFrom(ParseState *pstate,
/* Allocate workspace and zero all fields */
cstate = (CopyFromStateData *) palloc0(sizeof(CopyFromStateData));
+ edata = (CopyFromExecutionData *) palloc0(sizeof(CopyFromExecutionData));
+ cstate->edata = edata;
+
/*
* We allocate everything used by a cstate in a new memory context. This
* avoids memory leaks during repeated use of COPY in a query.
*/
- cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
- "COPY",
- ALLOCSET_DEFAULT_SIZES);
+ edata->copycontext = AllocSetContextCreate(CurrentMemoryContext,
+ "COPY",
+ ALLOCSET_DEFAULT_SIZES);
- oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ oldcontext = MemoryContextSwitchTo(edata->copycontext);
/* Extract options from the statement node tree */
ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options);
@@ -1617,19 +1625,19 @@ BeginCopyFrom(ParseState *pstate,
/* Set up soft error handler for ON_ERROR */
if (cstate->opts.on_error != COPY_ON_ERROR_STOP)
{
- cstate->escontext = makeNode(ErrorSaveContext);
- cstate->escontext->type = T_ErrorSaveContext;
- cstate->escontext->error_occurred = false;
+ edata->escontext = makeNode(ErrorSaveContext);
+ edata->escontext->type = T_ErrorSaveContext;
+ edata->escontext->error_occurred = false;
/*
* Currently we only support COPY_ON_ERROR_IGNORE. We'll add other
* options later
*/
if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
- cstate->escontext->details_wanted = false;
+ edata->escontext->details_wanted = false;
}
else
- cstate->escontext = NULL;
+ edata->escontext = NULL;
/* Convert FORCE_NULL name list to per-column flags, check validity */
cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
@@ -1663,7 +1671,7 @@ BeginCopyFrom(ParseState *pstate,
List *attnums;
ListCell *cur;
- cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
+ edata->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.convert_select);
@@ -1677,7 +1685,7 @@ BeginCopyFrom(ParseState *pstate,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg_internal("selected column \"%s\" not referenced by COPY",
NameStr(attr->attname))));
- cstate->convert_select_flags[attnum - 1] = true;
+ edata->convert_select_flags[attnum - 1] = true;
}
}
@@ -1694,14 +1702,14 @@ BeginCopyFrom(ParseState *pstate,
cstate->file_encoding == PG_SQL_ASCII ||
GetDatabaseEncoding() == PG_SQL_ASCII)
{
- cstate->need_transcoding = false;
+ cstate->edata->need_transcoding = false;
}
else
{
- cstate->need_transcoding = true;
- cstate->conversion_proc = FindDefaultConversionProc(cstate->file_encoding,
- GetDatabaseEncoding());
- if (!OidIsValid(cstate->conversion_proc))
+ cstate->edata->need_transcoding = true;
+ cstate->edata->conversion_proc = FindDefaultConversionProc(cstate->file_encoding,
+ GetDatabaseEncoding());
+ if (!OidIsValid(cstate->edata->conversion_proc))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_FUNCTION),
errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
@@ -1709,17 +1717,17 @@ BeginCopyFrom(ParseState *pstate,
pg_encoding_to_char(GetDatabaseEncoding()))));
}
- cstate->copy_src = COPY_FILE; /* default */
+ edata->copy_src = COPY_FILE; /* default */
- cstate->whereClause = whereClause;
+ edata->whereClause = whereClause;
/* Initialize state variables */
- cstate->eol_type = EOL_UNKNOWN;
- cstate->cur_relname = RelationGetRelationName(cstate->rel);
- cstate->cur_lineno = 0;
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
- cstate->relname_only = false;
+ edata->eol_type = EOL_UNKNOWN;
+ edata->cur_relname = RelationGetRelationName(cstate->rel);
+ edata->cur_lineno = 0;
+ edata->cur_attname = NULL;
+ edata->cur_attval = NULL;
+ edata->relname_only = false;
/*
* Allocate buffers for the input pipeline.
@@ -1727,17 +1735,17 @@ BeginCopyFrom(ParseState *pstate,
* attribute_buf and raw_buf are used in both text and binary modes, but
* input_buf and line_buf only in text mode.
*/
- cstate->raw_buf = palloc(RAW_BUF_SIZE + 1);
- cstate->raw_buf_index = cstate->raw_buf_len = 0;
- cstate->raw_reached_eof = false;
+ edata->raw_buf = palloc(RAW_BUF_SIZE + 1);
+ edata->raw_buf_index = edata->raw_buf_len = 0;
+ edata->raw_reached_eof = false;
- initStringInfo(&cstate->attribute_buf);
+ initStringInfo(&edata->attribute_buf);
/* Assign range table and rteperminfos, we'll need them in CopyFrom. */
if (pstate)
{
- cstate->range_table = pstate->p_rtable;
- cstate->rteperminfos = pstate->p_rteperminfos;
+ edata->range_table = pstate->p_rtable;
+ edata->rteperminfos = pstate->p_rteperminfos;
}
num_defaults = 0;
@@ -1818,26 +1826,26 @@ BeginCopyFrom(ParseState *pstate,
}
}
- cstate->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool));
+ edata->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool));
/* initialize progress */
pgstat_progress_start_command(PROGRESS_COMMAND_COPY,
cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid);
- cstate->bytes_processed = 0;
+ edata->bytes_processed = 0;
/* We keep those variables in cstate. */
cstate->in_functions = in_functions;
cstate->typioparams = typioparams;
- cstate->defmap = defmap;
- cstate->defexprs = defexprs;
- cstate->volatile_defexprs = volatile_defexprs;
- cstate->num_defaults = num_defaults;
+ edata->defmap = defmap;
+ edata->defexprs = defexprs;
+ edata->volatile_defexprs = volatile_defexprs;
+ edata->num_defaults = num_defaults;
cstate->is_program = is_program;
if (data_source_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_src = COPY_CALLBACK;
+ edata->copy_src = COPY_CALLBACK;
cstate->data_source_cb = data_source_cb;
}
else if (pipe)
@@ -1847,7 +1855,7 @@ BeginCopyFrom(ParseState *pstate,
if (whereToSendOutput == DestRemote)
ReceiveCopyBegin(cstate);
else
- cstate->copy_file = stdin;
+ edata->copy_file = stdin;
}
else
{
@@ -1856,8 +1864,8 @@ BeginCopyFrom(ParseState *pstate,
if (cstate->is_program)
{
progress_vals[1] = PROGRESS_COPY_TYPE_PROGRAM;
- cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
- if (cstate->copy_file == NULL)
+ edata->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
+ if (edata->copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not execute command \"%s\": %m",
@@ -1868,8 +1876,8 @@ BeginCopyFrom(ParseState *pstate,
struct stat st;
progress_vals[1] = PROGRESS_COPY_TYPE_FILE;
- cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
- if (cstate->copy_file == NULL)
+ edata->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
+ if (edata->copy_file == NULL)
{
/* copy errno because ereport subfunctions might change it */
int save_errno = errno;
@@ -1883,7 +1891,7 @@ BeginCopyFrom(ParseState *pstate,
"You may want a client-side facility such as psql's \\copy.") : 0));
}
- if (fstat(fileno(cstate->copy_file), &st))
+ if (fstat(fileno(edata->copy_file), &st))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m",
@@ -1923,7 +1931,7 @@ EndCopyFrom(CopyFromState cstate)
}
else
{
- if (cstate->filename != NULL && FreeFile(cstate->copy_file))
+ if (cstate->filename != NULL && FreeFile(cstate->edata->copy_file))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not close file \"%s\": %m",
@@ -1932,7 +1940,7 @@ EndCopyFrom(CopyFromState cstate)
pgstat_progress_end_command();
- MemoryContextDelete(cstate->copycontext);
+ MemoryContextDelete(cstate->edata->copycontext);
pfree(cstate);
}
@@ -1946,7 +1954,7 @@ ClosePipeFromProgram(CopyFromState cstate)
Assert(cstate->is_program);
- pclose_rc = ClosePipeStream(cstate->copy_file);
+ pclose_rc = ClosePipeStream(cstate->edata->copy_file);
if (pclose_rc == -1)
ereport(ERROR,
(errcode_for_file_access(),
@@ -1959,7 +1967,7 @@ ClosePipeFromProgram(CopyFromState cstate)
* should not report that as an error. Otherwise, SIGPIPE indicates a
* problem.
*/
- if (!cstate->raw_reached_eof &&
+ if (!cstate->edata->raw_reached_eof &&
wait_result_is_signal(pclose_rc, SIGPIPE))
return;
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index b1ae97b833d..595dc84b172 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -126,12 +126,12 @@ if (1) \
#define REFILL_LINEBUF \
if (1) \
{ \
- if (input_buf_ptr > cstate->input_buf_index) \
+ if (input_buf_ptr > edata->input_buf_index) \
{ \
- appendBinaryStringInfo(&cstate->line_buf, \
- cstate->input_buf + cstate->input_buf_index, \
- input_buf_ptr - cstate->input_buf_index); \
- cstate->input_buf_index = input_buf_ptr; \
+ appendBinaryStringInfo(&edata->line_buf, \
+ edata->input_buf + edata->input_buf_index, \
+ input_buf_ptr - edata->input_buf_index); \
+ edata->input_buf_index = input_buf_ptr; \
} \
} else ((void) 0)
@@ -180,8 +180,8 @@ ReceiveCopyBegin(CopyFromState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_src = COPY_FRONTEND;
- cstate->fe_msgbuf = makeStringInfo();
+ cstate->edata->copy_src = COPY_FRONTEND;
+ cstate->edata->fe_msgbuf = makeStringInfo();
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
}
@@ -246,23 +246,23 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
{
int bytesread = 0;
- switch (cstate->copy_src)
+ switch (cstate->edata->copy_src)
{
case COPY_FILE:
- bytesread = fread(databuf, 1, maxread, cstate->copy_file);
- if (ferror(cstate->copy_file))
+ bytesread = fread(databuf, 1, maxread, cstate->edata->copy_file);
+ if (ferror(cstate->edata->copy_file))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
if (bytesread == 0)
- cstate->raw_reached_eof = true;
+ cstate->edata->raw_reached_eof = true;
break;
case COPY_FRONTEND:
- while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
+ while (maxread > 0 && bytesread < minread && !cstate->edata->raw_reached_eof)
{
int avail;
- while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
+ while (cstate->edata->fe_msgbuf->cursor >= cstate->edata->fe_msgbuf->len)
{
/* Try to receive another message */
int mtype;
@@ -297,7 +297,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
}
/* Now collect the message body */
- if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
+ if (pq_getmessage(cstate->edata->fe_msgbuf, maxmsglen))
ereport(ERROR,
(errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("unexpected EOF on client connection with an open transaction")));
@@ -309,13 +309,13 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case PqMsg_CopyDone:
/* COPY IN correctly terminated by frontend */
- cstate->raw_reached_eof = true;
+ cstate->edata->raw_reached_eof = true;
return bytesread;
case PqMsg_CopyFail:
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
errmsg("COPY from stdin failed: %s",
- pq_getmsgstring(cstate->fe_msgbuf))));
+ pq_getmsgstring(cstate->edata->fe_msgbuf))));
break;
case PqMsg_Flush:
case PqMsg_Sync:
@@ -331,10 +331,10 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
Assert(false); /* NOT REACHED */
}
}
- avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
+ avail = cstate->edata->fe_msgbuf->len - cstate->edata->fe_msgbuf->cursor;
if (avail > maxread)
avail = maxread;
- pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
+ pq_copymsgbytes(cstate->edata->fe_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
maxread -= avail;
bytesread += avail;
@@ -399,12 +399,14 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
static void
CopyConvertBuf(CopyFromState cstate)
{
+ CopyFromExecutionData *edata = cstate->edata;
+
/*
* If the file and server encoding are the same, no encoding conversion is
* required. However, we still need to verify that the input is valid for
* the encoding.
*/
- if (!cstate->need_transcoding)
+ if (!edata->need_transcoding)
{
/*
* When conversion is not required, input_buf and raw_buf are the
@@ -412,8 +414,8 @@ CopyConvertBuf(CopyFromState cstate)
* input_buf_len tracks how many of those bytes have already been
* verified.
*/
- int preverifiedlen = cstate->input_buf_len;
- int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
+ int preverifiedlen = edata->input_buf_len;
+ int unverifiedlen = edata->raw_buf_len - edata->input_buf_len;
int nverified;
if (unverifiedlen == 0)
@@ -421,8 +423,8 @@ CopyConvertBuf(CopyFromState cstate)
/*
* If no more raw data is coming, report the EOF to the caller.
*/
- if (cstate->raw_reached_eof)
- cstate->input_reached_eof = true;
+ if (edata->raw_reached_eof)
+ edata->input_reached_eof = true;
return;
}
@@ -431,7 +433,7 @@ CopyConvertBuf(CopyFromState cstate)
* previous round.
*/
nverified = pg_encoding_verifymbstr(cstate->file_encoding,
- cstate->raw_buf + preverifiedlen,
+ edata->raw_buf + preverifiedlen,
unverifiedlen);
if (nverified == 0)
{
@@ -444,11 +446,11 @@ CopyConvertBuf(CopyFromState cstate)
* least one character, and a failure to do so means that we've
* hit an invalid byte sequence.
*/
- if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
- cstate->input_reached_error = true;
+ if (edata->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
+ edata->input_reached_error = true;
return;
}
- cstate->input_buf_len += nverified;
+ edata->input_buf_len += nverified;
}
else
{
@@ -462,31 +464,31 @@ CopyConvertBuf(CopyFromState cstate)
int dstlen;
int convertedlen;
- if (RAW_BUF_BYTES(cstate) == 0)
+ if (RAW_BUF_BYTES(edata) == 0)
{
/*
* If no more raw data is coming, report the EOF to the caller.
*/
- if (cstate->raw_reached_eof)
- cstate->input_reached_eof = true;
+ if (edata->raw_reached_eof)
+ edata->input_reached_eof = true;
return;
}
/*
* First, copy down any unprocessed data.
*/
- nbytes = INPUT_BUF_BYTES(cstate);
- if (nbytes > 0 && cstate->input_buf_index > 0)
- memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
+ nbytes = INPUT_BUF_BYTES(edata);
+ if (nbytes > 0 && edata->input_buf_index > 0)
+ memmove(edata->input_buf, edata->input_buf + edata->input_buf_index,
nbytes);
- cstate->input_buf_index = 0;
- cstate->input_buf_len = nbytes;
- cstate->input_buf[nbytes] = '\0';
+ edata->input_buf_index = 0;
+ edata->input_buf_len = nbytes;
+ edata->input_buf[nbytes] = '\0';
- src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
- srclen = cstate->raw_buf_len - cstate->raw_buf_index;
- dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
- dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
+ src = (unsigned char *) edata->raw_buf + edata->raw_buf_index;
+ srclen = edata->raw_buf_len - edata->raw_buf_index;
+ dst = (unsigned char *) edata->input_buf + edata->input_buf_len;
+ dstlen = INPUT_BUF_SIZE - edata->input_buf_len + 1;
/*
* Do the conversion. This might stop short, if there is an invalid
@@ -501,7 +503,7 @@ CopyConvertBuf(CopyFromState cstate)
* after the end-of-input marker as long as it's valid for the
* encoding, but that's harmless.
*/
- convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
+ convertedlen = pg_do_encoding_conversion_buf(cstate->edata->conversion_proc,
cstate->file_encoding,
GetDatabaseEncoding(),
src, srclen,
@@ -517,12 +519,12 @@ CopyConvertBuf(CopyFromState cstate)
* failure to do so must mean that we've hit a byte sequence
* that's invalid.
*/
- if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
- cstate->input_reached_error = true;
+ if (edata->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
+ edata->input_reached_error = true;
return;
}
- cstate->raw_buf_index += convertedlen;
- cstate->input_buf_len += strlen((char *) dst);
+ edata->raw_buf_index += convertedlen;
+ edata->input_buf_len += strlen((char *) dst);
}
}
@@ -532,18 +534,20 @@ CopyConvertBuf(CopyFromState cstate)
static void
CopyConversionError(CopyFromState cstate)
{
- Assert(cstate->raw_buf_len > 0);
- Assert(cstate->input_reached_error);
+ CopyFromExecutionData *edata = cstate->edata;
+
+ Assert(edata->raw_buf_len > 0);
+ Assert(edata->input_reached_error);
- if (!cstate->need_transcoding)
+ if (!edata->need_transcoding)
{
/*
* Everything up to input_buf_len was successfully verified, and
* input_buf_len points to the invalid or incomplete character.
*/
report_invalid_encoding(cstate->file_encoding,
- cstate->raw_buf + cstate->input_buf_len,
- cstate->raw_buf_len - cstate->input_buf_len);
+ edata->raw_buf + edata->input_buf_len,
+ edata->raw_buf_len - edata->input_buf_len);
}
else
{
@@ -560,12 +564,12 @@ CopyConversionError(CopyFromState cstate)
unsigned char *dst;
int dstlen;
- src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
- srclen = cstate->raw_buf_len - cstate->raw_buf_index;
- dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
- dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
+ src = (unsigned char *) edata->raw_buf + edata->raw_buf_index;
+ srclen = edata->raw_buf_len - edata->raw_buf_index;
+ dst = (unsigned char *) edata->input_buf + edata->input_buf_len;
+ dstlen = INPUT_BUF_SIZE - edata->input_buf_len + 1;
- (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
+ (void) pg_do_encoding_conversion_buf(cstate->edata->conversion_proc,
cstate->file_encoding,
GetDatabaseEncoding(),
src, srclen,
@@ -589,6 +593,7 @@ CopyConversionError(CopyFromState cstate)
static void
CopyLoadRawBuf(CopyFromState cstate)
{
+ CopyFromExecutionData *edata = cstate->edata;
int nbytes;
int inbytes;
@@ -596,45 +601,45 @@ CopyLoadRawBuf(CopyFromState cstate)
* In text mode, if encoding conversion is not required, raw_buf and
* input_buf point to the same buffer. Their len/index better agree, too.
*/
- if (cstate->raw_buf == cstate->input_buf)
+ if (edata->raw_buf == edata->input_buf)
{
- Assert(!cstate->need_transcoding);
- Assert(cstate->raw_buf_index == cstate->input_buf_index);
- Assert(cstate->input_buf_len <= cstate->raw_buf_len);
+ Assert(!edata->need_transcoding);
+ Assert(edata->raw_buf_index == edata->input_buf_index);
+ Assert(edata->input_buf_len <= edata->raw_buf_len);
}
/*
* Copy down the unprocessed data if any.
*/
- nbytes = RAW_BUF_BYTES(cstate);
- if (nbytes > 0 && cstate->raw_buf_index > 0)
- memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
+ nbytes = RAW_BUF_BYTES(edata);
+ if (nbytes > 0 && edata->raw_buf_index > 0)
+ memmove(edata->raw_buf, edata->raw_buf + edata->raw_buf_index,
nbytes);
- cstate->raw_buf_len -= cstate->raw_buf_index;
- cstate->raw_buf_index = 0;
+ edata->raw_buf_len -= edata->raw_buf_index;
+ edata->raw_buf_index = 0;
/*
* If raw_buf and input_buf are in fact the same buffer, adjust the
* input_buf variables, too.
*/
- if (cstate->raw_buf == cstate->input_buf)
+ if (edata->raw_buf == edata->input_buf)
{
- cstate->input_buf_len -= cstate->input_buf_index;
- cstate->input_buf_index = 0;
+ edata->input_buf_len -= edata->input_buf_index;
+ edata->input_buf_index = 0;
}
/* Load more data */
- inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
- 1, RAW_BUF_SIZE - cstate->raw_buf_len);
+ inbytes = CopyGetData(cstate, edata->raw_buf + edata->raw_buf_len,
+ 1, RAW_BUF_SIZE - edata->raw_buf_len);
nbytes += inbytes;
- cstate->raw_buf[nbytes] = '\0';
- cstate->raw_buf_len = nbytes;
+ edata->raw_buf[nbytes] = '\0';
+ edata->raw_buf_len = nbytes;
- cstate->bytes_processed += inbytes;
- pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
+ edata->bytes_processed += inbytes;
+ pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, edata->bytes_processed);
if (inbytes == 0)
- cstate->raw_reached_eof = true;
+ edata->raw_reached_eof = true;
}
/*
@@ -643,24 +648,25 @@ CopyLoadRawBuf(CopyFromState cstate)
* On return, at least one more input character is loaded into
* input_buf, or input_reached_eof is set.
*
- * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
+ * If INPUT_BUF_BYTES(edata) > 0, the unprocessed bytes are moved to the start
* of the buffer and then we load more data after that.
*/
static void
CopyLoadInputBuf(CopyFromState cstate)
{
- int nbytes = INPUT_BUF_BYTES(cstate);
+ CopyFromExecutionData *edata = cstate->edata;
+ int nbytes = INPUT_BUF_BYTES(edata);
/*
* The caller has updated input_buf_index to indicate how much of the
* input has been consumed and isn't needed anymore. If input_buf is the
* same physical area as raw_buf, update raw_buf_index accordingly.
*/
- if (cstate->raw_buf == cstate->input_buf)
+ if (edata->raw_buf == edata->input_buf)
{
- Assert(!cstate->need_transcoding);
- Assert(cstate->input_buf_index >= cstate->raw_buf_index);
- cstate->raw_buf_index = cstate->input_buf_index;
+ Assert(!edata->need_transcoding);
+ Assert(edata->input_buf_index >= edata->raw_buf_index);
+ edata->raw_buf_index = edata->input_buf_index;
}
for (;;)
@@ -669,7 +675,7 @@ CopyLoadInputBuf(CopyFromState cstate)
CopyConvertBuf(cstate);
/* If we now have some more input bytes ready, return them */
- if (INPUT_BUF_BYTES(cstate) > nbytes)
+ if (INPUT_BUF_BYTES(edata) > nbytes)
return;
/*
@@ -677,15 +683,15 @@ CopyLoadInputBuf(CopyFromState cstate)
* multi-byte character but there is no more raw input data, report
* conversion error.
*/
- if (cstate->input_reached_error)
+ if (edata->input_reached_error)
CopyConversionError(cstate);
/* no more input, and everything has been converted */
- if (cstate->input_reached_eof)
+ if (edata->input_reached_eof)
break;
/* Try to load more raw data */
- Assert(!cstate->raw_reached_eof);
+ Assert(!edata->raw_reached_eof);
CopyLoadRawBuf(cstate);
}
}
@@ -700,13 +706,14 @@ CopyLoadInputBuf(CopyFromState cstate)
static int
CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
{
+ CopyFromExecutionData *edata = cstate->edata;
int copied_bytes = 0;
- if (RAW_BUF_BYTES(cstate) >= nbytes)
+ if (RAW_BUF_BYTES(edata) >= nbytes)
{
/* Enough bytes are present in the buffer. */
- memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
- cstate->raw_buf_index += nbytes;
+ memcpy(dest, edata->raw_buf + edata->raw_buf_index, nbytes);
+ edata->raw_buf_index += nbytes;
copied_bytes = nbytes;
}
else
@@ -720,17 +727,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
int copy_bytes;
/* Load more data if buffer is empty. */
- if (RAW_BUF_BYTES(cstate) == 0)
+ if (RAW_BUF_BYTES(edata) == 0)
{
CopyLoadRawBuf(cstate);
- if (cstate->raw_reached_eof)
+ if (edata->raw_reached_eof)
break; /* EOF */
}
/* Transfer some bytes. */
- copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
- memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
- cstate->raw_buf_index += copy_bytes;
+ copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(edata));
+ memcpy(dest, edata->raw_buf + edata->raw_buf_index, copy_bytes);
+ edata->raw_buf_index += copy_bytes;
dest += copy_bytes;
copied_bytes += copy_bytes;
} while (copied_bytes < nbytes);
@@ -770,6 +777,7 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
static pg_attribute_always_inline bool
NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
{
+ CopyFromExecutionData *edata = cstate->edata;
int fldct;
bool done = false;
@@ -777,7 +785,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
Assert(!cstate->opts.binary);
/* on input check that the header line is correct if needed */
- if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
+ if (edata->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
{
ListCell *cur;
TupleDesc tupDesc;
@@ -791,7 +799,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
for (int i = 0; i < lines_to_skip; i++)
{
- cstate->cur_lineno++;
+ edata->cur_lineno++;
if ((done = CopyReadLine(cstate, is_csv)))
break;
}
@@ -818,9 +826,9 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
char *colName;
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
- Assert(fldnum < cstate->max_fields);
+ Assert(fldnum < edata->max_fields);
- colName = cstate->raw_fields[fldnum++];
+ colName = edata->raw_fields[fldnum++];
if (colName == NULL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -841,7 +849,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
return false;
}
- cstate->cur_lineno++;
+ edata->cur_lineno++;
/* Actually read the line into memory here */
done = CopyReadLine(cstate, is_csv);
@@ -851,7 +859,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
* characters, we act as though it was newline followed by EOF, ie,
* process the line and then exit loop on next iteration.
*/
- if (done && cstate->line_buf.len == 0)
+ if (done && edata->line_buf.len == 0)
return false;
/* Parse the line into de-escaped field values */
@@ -860,7 +868,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
else
fldct = CopyReadAttributesText(cstate);
- *fields = cstate->raw_fields;
+ *fields = edata->raw_fields;
*nfields = fldct;
return true;
}
@@ -882,10 +890,10 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
{
TupleDesc tupDesc;
AttrNumber num_phys_attrs,
- num_defaults = cstate->num_defaults;
+ num_defaults = cstate->edata->num_defaults;
int i;
- int *defmap = cstate->defmap;
- ExprState **defexprs = cstate->defexprs;
+ int *defmap = cstate->edata->defmap;
+ ExprState **defexprs = cstate->edata->defexprs;
tupDesc = RelationGetDescr(cstate->rel);
num_phys_attrs = tupDesc->natts;
@@ -893,7 +901,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ MemSet(cstate->edata->defaults, false, num_phys_attrs * sizeof(bool));
/* Get one row from source */
if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
@@ -946,11 +954,12 @@ static pg_attribute_always_inline bool
CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
Datum *values, bool *nulls, bool is_csv)
{
+ CopyFromExecutionData *edata = cstate->edata;
TupleDesc tupDesc;
AttrNumber attr_count;
FmgrInfo *in_functions = cstate->in_functions;
Oid *typioparams = cstate->typioparams;
- ExprState **defexprs = cstate->defexprs;
+ ExprState **defexprs = edata->defexprs;
char **field_strings;
ListCell *cur;
int fldct;
@@ -986,8 +995,8 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
NameStr(att->attname))));
string = field_strings[fieldno++];
- if (cstate->convert_select_flags &&
- !cstate->convert_select_flags[m])
+ if (edata->convert_select_flags &&
+ !edata->convert_select_flags[m])
{
/* ignore input field, leaving column as NULL */
continue;
@@ -1017,13 +1026,13 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
}
}
- cstate->cur_attname = NameStr(att->attname);
- cstate->cur_attval = string;
+ edata->cur_attname = NameStr(att->attname);
+ edata->cur_attval = string;
if (string != NULL)
nulls[m] = false;
- if (cstate->defaults[m])
+ if (edata->defaults[m])
{
/* We must have switched into the per-tuple memory context */
Assert(econtext != NULL);
@@ -1039,12 +1048,12 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
string,
typioparams[m],
att->atttypmod,
- (Node *) cstate->escontext,
+ (Node *) edata->escontext,
&values[m]))
{
Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
- cstate->num_errors++;
+ edata->num_errors++;
if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
{
@@ -1053,36 +1062,36 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
* notice message, we suppress error context information other
* than the relation name.
*/
- Assert(!cstate->relname_only);
- cstate->relname_only = true;
+ Assert(!edata->relname_only);
+ edata->relname_only = true;
- if (cstate->cur_attval)
+ if (edata->cur_attval)
{
char *attval;
- attval = CopyLimitPrintoutLength(cstate->cur_attval);
+ attval = CopyLimitPrintoutLength(edata->cur_attval);
ereport(NOTICE,
errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
- cstate->cur_lineno,
- cstate->cur_attname,
+ edata->cur_lineno,
+ edata->cur_attname,
attval));
pfree(attval);
}
else
ereport(NOTICE,
errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
- cstate->cur_lineno,
- cstate->cur_attname));
+ edata->cur_lineno,
+ edata->cur_attname));
/* reset relname_only */
- cstate->relname_only = false;
+ edata->relname_only = false;
}
return true;
}
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
+ edata->cur_attname = NULL;
+ edata->cur_attval = NULL;
}
Assert(fieldno == attr_count);
@@ -1105,7 +1114,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
tupDesc = RelationGetDescr(cstate->rel);
attr_count = list_length(cstate->attnumlist);
- cstate->cur_lineno++;
+ cstate->edata->cur_lineno++;
if (!CopyGetInt16(cstate, &fld_count))
{
@@ -1144,13 +1153,13 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
int m = attnum - 1;
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
- cstate->cur_attname = NameStr(att->attname);
+ cstate->edata->cur_attname = NameStr(att->attname);
values[m] = CopyReadBinaryAttribute(cstate,
&in_functions[m],
typioparams[m],
att->atttypmod,
&nulls[m]);
- cstate->cur_attname = NULL;
+ cstate->edata->cur_attname = NULL;
}
return true;
@@ -1166,10 +1175,11 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
static bool
CopyReadLine(CopyFromState cstate, bool is_csv)
{
+ CopyFromExecutionData *edata = cstate->edata;
bool result;
- resetStringInfo(&cstate->line_buf);
- cstate->line_buf_valid = false;
+ resetStringInfo(&edata->line_buf);
+ edata->line_buf_valid = false;
/* Parse data and transfer into line_buf */
result = CopyReadLineText(cstate, is_csv);
@@ -1181,19 +1191,19 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (edata->copy_src == COPY_FRONTEND)
{
int inbytes;
do
{
- inbytes = CopyGetData(cstate, cstate->input_buf,
+ inbytes = CopyGetData(cstate, edata->input_buf,
1, INPUT_BUF_SIZE);
} while (inbytes > 0);
- cstate->input_buf_index = 0;
- cstate->input_buf_len = 0;
- cstate->raw_buf_index = 0;
- cstate->raw_buf_len = 0;
+ edata->input_buf_index = 0;
+ edata->input_buf_len = 0;
+ edata->raw_buf_index = 0;
+ edata->raw_buf_len = 0;
}
}
else
@@ -1202,26 +1212,26 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* If we didn't hit EOF, then we must have transferred the EOL marker
* to line_buf along with the data. Get rid of it.
*/
- switch (cstate->eol_type)
+ switch (edata->eol_type)
{
case EOL_NL:
- Assert(cstate->line_buf.len >= 1);
- Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
- cstate->line_buf.len--;
- cstate->line_buf.data[cstate->line_buf.len] = '\0';
+ Assert(edata->line_buf.len >= 1);
+ Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\n');
+ edata->line_buf.len--;
+ edata->line_buf.data[edata->line_buf.len] = '\0';
break;
case EOL_CR:
- Assert(cstate->line_buf.len >= 1);
- Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
- cstate->line_buf.len--;
- cstate->line_buf.data[cstate->line_buf.len] = '\0';
+ Assert(edata->line_buf.len >= 1);
+ Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\r');
+ edata->line_buf.len--;
+ edata->line_buf.data[edata->line_buf.len] = '\0';
break;
case EOL_CRNL:
- Assert(cstate->line_buf.len >= 2);
- Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
- Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
- cstate->line_buf.len -= 2;
- cstate->line_buf.data[cstate->line_buf.len] = '\0';
+ Assert(edata->line_buf.len >= 2);
+ Assert(edata->line_buf.data[edata->line_buf.len - 2] == '\r');
+ Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\n');
+ edata->line_buf.len -= 2;
+ edata->line_buf.data[edata->line_buf.len] = '\0';
break;
case EOL_UNKNOWN:
/* shouldn't get here */
@@ -1231,7 +1241,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
}
/* Now it's safe to use the buffer in error messages */
- cstate->line_buf_valid = true;
+ edata->line_buf_valid = true;
return result;
}
@@ -1242,6 +1252,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
static bool
CopyReadLineText(CopyFromState cstate, bool is_csv)
{
+ CopyFromExecutionData *edata = cstate->edata;
char *copy_input_buf;
int input_buf_ptr;
int copy_buf_len;
@@ -1289,9 +1300,9 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
* For a little extra speed within the loop, we copy input_buf and
* input_buf_len into local variables.
*/
- copy_input_buf = cstate->input_buf;
- input_buf_ptr = cstate->input_buf_index;
- copy_buf_len = cstate->input_buf_len;
+ copy_input_buf = edata->input_buf;
+ input_buf_ptr = edata->input_buf_index;
+ copy_buf_len = edata->input_buf_len;
for (;;)
{
@@ -1312,15 +1323,15 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
CopyLoadInputBuf(cstate);
/* update our local variables */
- hit_eof = cstate->input_reached_eof;
- input_buf_ptr = cstate->input_buf_index;
- copy_buf_len = cstate->input_buf_len;
+ hit_eof = edata->input_reached_eof;
+ input_buf_ptr = edata->input_buf_index;
+ copy_buf_len = edata->input_buf_len;
/*
* If we are completely out of data, break out of the loop,
* reporting EOF.
*/
- if (INPUT_BUF_BYTES(cstate) <= 0)
+ if (INPUT_BUF_BYTES(edata) <= 0)
{
result = true;
break;
@@ -1366,16 +1377,16 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
* best we can do. (XXX it's arguable whether we should do this
* at all --- is cur_lineno a physical or logical count?)
*/
- if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
- cstate->cur_lineno++;
+ if (in_quote && c == (edata->eol_type == EOL_NL ? '\n' : '\r'))
+ edata->cur_lineno++;
}
/* Process \r */
if (c == '\r' && (!is_csv || !in_quote))
{
/* Check for \r\n on first line, _and_ handle \r\n. */
- if (cstate->eol_type == EOL_UNKNOWN ||
- cstate->eol_type == EOL_CRNL)
+ if (edata->eol_type == EOL_UNKNOWN ||
+ edata->eol_type == EOL_CRNL)
{
/*
* If need more data, go back to loop top to load it.
@@ -1391,12 +1402,12 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
if (c == '\n')
{
input_buf_ptr++; /* eat newline */
- cstate->eol_type = EOL_CRNL; /* in case not set yet */
+ edata->eol_type = EOL_CRNL; /* in case not set yet */
}
else
{
/* found \r, but no \n */
- if (cstate->eol_type == EOL_CRNL)
+ if (edata->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
!is_csv ?
@@ -1410,10 +1421,10 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
* if we got here, it is the first line and we didn't find
* \n, so don't consume the peeked character
*/
- cstate->eol_type = EOL_CR;
+ edata->eol_type = EOL_CR;
}
}
- else if (cstate->eol_type == EOL_NL)
+ else if (edata->eol_type == EOL_NL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
!is_csv ?
@@ -1429,7 +1440,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
/* Process \n */
if (c == '\n' && (!is_csv || !in_quote))
{
- if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
+ if (edata->eol_type == EOL_CR || edata->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
!is_csv ?
@@ -1438,7 +1449,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
!is_csv ?
errhint("Use \"\\n\" to represent newline.") :
errhint("Use quoted CSV field to represent newline.")));
- cstate->eol_type = EOL_NL; /* in case not set yet */
+ edata->eol_type = EOL_NL; /* in case not set yet */
/* If reach here, we have found the line terminator */
break;
}
@@ -1465,7 +1476,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
if (c2 == '.')
{
input_buf_ptr++; /* consume the '.' */
- if (cstate->eol_type == EOL_CRNL)
+ if (edata->eol_type == EOL_CRNL)
{
/* Get the next character */
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
@@ -1492,9 +1503,9 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("end-of-copy marker is not alone on its line")));
- if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
- (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
- (cstate->eol_type == EOL_CR && c2 != '\r'))
+ if ((edata->eol_type == EOL_NL && c2 != '\n') ||
+ (edata->eol_type == EOL_CRNL && c2 != '\n') ||
+ (edata->eol_type == EOL_CR && c2 != '\r'))
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("end-of-copy marker does not match previous newline style")));
@@ -1502,8 +1513,8 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
/*
* If there is any data on this line before the \., complain.
*/
- if (cstate->line_buf.len > 0 ||
- prev_raw_ptr > cstate->input_buf_index)
+ if (edata->line_buf.len > 0 ||
+ prev_raw_ptr > edata->input_buf_index)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("end-of-copy marker is not alone on its line")));
@@ -1511,7 +1522,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
/*
* Discard the \. and newline, then report EOF.
*/
- cstate->input_buf_index = input_buf_ptr;
+ edata->input_buf_index = input_buf_ptr;
result = true; /* report EOF */
break;
}
@@ -1572,6 +1583,7 @@ GetDecimalFromHex(char hex)
static int
CopyReadAttributesText(CopyFromState cstate)
{
+ CopyFromExecutionData *edata = cstate->edata;
char delimc = cstate->opts.delim[0];
int fieldno;
char *output_ptr;
@@ -1582,31 +1594,31 @@ CopyReadAttributesText(CopyFromState cstate)
* We need a special case for zero-column tables: check that the input
* line is empty, and return.
*/
- if (cstate->max_fields <= 0)
+ if (edata->max_fields <= 0)
{
- if (cstate->line_buf.len != 0)
+ if (edata->line_buf.len != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("extra data after last expected column")));
return 0;
}
- resetStringInfo(&cstate->attribute_buf);
+ resetStringInfo(&edata->attribute_buf);
/*
* The de-escaped attributes will certainly not be longer than the input
* data line, so we can just force attribute_buf to be large enough and
* then transfer data without any checks for enough space. We need to do
* it this way because enlarging attribute_buf mid-stream would invalidate
- * pointers already stored into cstate->raw_fields[].
+ * pointers already stored into edata->raw_fields[].
*/
- if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
- enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
- output_ptr = cstate->attribute_buf.data;
+ if (edata->attribute_buf.maxlen <= edata->line_buf.len)
+ enlargeStringInfo(&edata->attribute_buf, edata->line_buf.len);
+ output_ptr = edata->attribute_buf.data;
/* set pointer variables for loop */
- cur_ptr = cstate->line_buf.data;
- line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
+ cur_ptr = edata->line_buf.data;
+ line_end_ptr = edata->line_buf.data + edata->line_buf.len;
/* Outer loop iterates over fields */
fieldno = 0;
@@ -1619,16 +1631,16 @@ CopyReadAttributesText(CopyFromState cstate)
bool saw_non_ascii = false;
/* Make sure there is enough space for the next value */
- if (fieldno >= cstate->max_fields)
+ if (fieldno >= edata->max_fields)
{
- cstate->max_fields *= 2;
- cstate->raw_fields =
- repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
+ edata->max_fields *= 2;
+ edata->raw_fields =
+ repalloc(edata->raw_fields, edata->max_fields * sizeof(char *));
}
/* Remember start of field on both input and output sides */
start_ptr = cur_ptr;
- cstate->raw_fields[fieldno] = output_ptr;
+ edata->raw_fields[fieldno] = output_ptr;
/*
* Scan data for field.
@@ -1757,7 +1769,7 @@ CopyReadAttributesText(CopyFromState cstate)
input_len = end_ptr - start_ptr;
if (input_len == cstate->opts.null_print_len &&
strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
- cstate->raw_fields[fieldno] = NULL;
+ edata->raw_fields[fieldno] = NULL;
/* Check whether raw input matched default marker */
else if (fieldno < list_length(cstate->attnumlist) &&
cstate->opts.default_print &&
@@ -1767,10 +1779,10 @@ CopyReadAttributesText(CopyFromState cstate)
/* fieldno is 0-indexed and attnum is 1-indexed */
int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
- if (cstate->defexprs[m] != NULL)
+ if (edata->defexprs[m] != NULL)
{
/* defaults contain entries for all physical attributes */
- cstate->defaults[m] = true;
+ edata->defaults[m] = true;
}
else
{
@@ -1794,7 +1806,7 @@ CopyReadAttributesText(CopyFromState cstate)
*/
if (saw_non_ascii)
{
- char *fld = cstate->raw_fields[fieldno];
+ char *fld = edata->raw_fields[fieldno];
pg_verifymbstr(fld, output_ptr - fld, false);
}
@@ -1812,7 +1824,7 @@ CopyReadAttributesText(CopyFromState cstate)
/* Clean up state of attribute_buf */
output_ptr--;
Assert(*output_ptr == '\0');
- cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
+ edata->attribute_buf.len = (output_ptr - edata->attribute_buf.data);
return fieldno;
}
@@ -1826,6 +1838,7 @@ CopyReadAttributesText(CopyFromState cstate)
static int
CopyReadAttributesCSV(CopyFromState cstate)
{
+ CopyFromExecutionData *edata = cstate->edata;
char delimc = cstate->opts.delim[0];
char quotec = cstate->opts.quote[0];
char escapec = cstate->opts.escape[0];
@@ -1838,31 +1851,31 @@ CopyReadAttributesCSV(CopyFromState cstate)
* We need a special case for zero-column tables: check that the input
* line is empty, and return.
*/
- if (cstate->max_fields <= 0)
+ if (edata->max_fields <= 0)
{
- if (cstate->line_buf.len != 0)
+ if (edata->line_buf.len != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("extra data after last expected column")));
return 0;
}
- resetStringInfo(&cstate->attribute_buf);
+ resetStringInfo(&edata->attribute_buf);
/*
* The de-escaped attributes will certainly not be longer than the input
* data line, so we can just force attribute_buf to be large enough and
* then transfer data without any checks for enough space. We need to do
* it this way because enlarging attribute_buf mid-stream would invalidate
- * pointers already stored into cstate->raw_fields[].
+ * pointers already stored into edata->raw_fields[].
*/
- if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
- enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
- output_ptr = cstate->attribute_buf.data;
+ if (edata->attribute_buf.maxlen <= edata->line_buf.len)
+ enlargeStringInfo(&edata->attribute_buf, edata->line_buf.len);
+ output_ptr = edata->attribute_buf.data;
/* set pointer variables for loop */
- cur_ptr = cstate->line_buf.data;
- line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
+ cur_ptr = edata->line_buf.data;
+ line_end_ptr = edata->line_buf.data + edata->line_buf.len;
/* Outer loop iterates over fields */
fieldno = 0;
@@ -1875,16 +1888,16 @@ CopyReadAttributesCSV(CopyFromState cstate)
int input_len;
/* Make sure there is enough space for the next value */
- if (fieldno >= cstate->max_fields)
+ if (fieldno >= edata->max_fields)
{
- cstate->max_fields *= 2;
- cstate->raw_fields =
- repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
+ edata->max_fields *= 2;
+ edata->raw_fields =
+ repalloc(edata->raw_fields, edata->max_fields * sizeof(char *));
}
/* Remember start of field on both input and output sides */
start_ptr = cur_ptr;
- cstate->raw_fields[fieldno] = output_ptr;
+ edata->raw_fields[fieldno] = output_ptr;
/*
* Scan data for field,
@@ -1972,7 +1985,7 @@ endfield:
input_len = end_ptr - start_ptr;
if (!saw_quote && input_len == cstate->opts.null_print_len &&
strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
- cstate->raw_fields[fieldno] = NULL;
+ edata->raw_fields[fieldno] = NULL;
/* Check whether raw input matched default marker */
else if (fieldno < list_length(cstate->attnumlist) &&
cstate->opts.default_print &&
@@ -1982,10 +1995,10 @@ endfield:
/* fieldno is 0-index and attnum is 1-index */
int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
- if (cstate->defexprs[m] != NULL)
+ if (edata->defexprs[m] != NULL)
{
/* defaults contain entries for all physical attributes */
- cstate->defaults[m] = true;
+ edata->defaults[m] = true;
}
else
{
@@ -2009,7 +2022,7 @@ endfield:
/* Clean up state of attribute_buf */
output_ptr--;
Assert(*output_ptr == '\0');
- cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
+ edata->attribute_buf.len = (output_ptr - edata->attribute_buf.data);
return fieldno;
}
@@ -2023,6 +2036,7 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
Oid typioparam, int32 typmod,
bool *isnull)
{
+ CopyFromExecutionData *edata = cstate->edata;
int32 fld_size;
Datum result;
@@ -2041,24 +2055,24 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
errmsg("invalid field size")));
/* reset attribute_buf to empty, and load raw data in it */
- resetStringInfo(&cstate->attribute_buf);
+ resetStringInfo(&edata->attribute_buf);
- enlargeStringInfo(&cstate->attribute_buf, fld_size);
- if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
+ enlargeStringInfo(&edata->attribute_buf, fld_size);
+ if (CopyReadBinaryData(cstate, edata->attribute_buf.data,
fld_size) != fld_size)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unexpected EOF in COPY data")));
- cstate->attribute_buf.len = fld_size;
- cstate->attribute_buf.data[fld_size] = '\0';
+ edata->attribute_buf.len = fld_size;
+ edata->attribute_buf.data[fld_size] = '\0';
/* Call the column type's binary input converter */
- result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
+ result = ReceiveFunctionCall(flinfo, &edata->attribute_buf,
typioparam, typmod);
/* Trouble if it didn't eat the whole buffer */
- if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
+ if (edata->attribute_buf.cursor != edata->attribute_buf.len)
ereport(ERROR,
(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
errmsg("incorrect binary data format")));
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 67b94b91cae..e90fab6d958 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -62,40 +62,24 @@ typedef enum CopyDest
* invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
* when we have to do it the hard way.
*/
-typedef struct CopyToStateData
+typedef struct CopyToExecutionData
{
- /* format-specific routines */
- const CopyToRoutine *routine;
-
/* low-level state data */
CopyDest copy_dest; /* type of copy source/destination */
FILE *copy_file; /* used if copy_dest == COPY_FILE */
StringInfo fe_msgbuf; /* used for all dests during COPY TO */
- int file_encoding; /* file or remote side's character encoding */
bool need_transcoding; /* file encoding diff from server? */
bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
-
/*
* Working state
*/
MemoryContext copycontext; /* per-copy execution context */
- FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
+} CopyToExecutionData;
/* DestReceiver for COPY (query) TO */
typedef struct
@@ -193,7 +177,7 @@ CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc)
* For non-binary copy, we need to convert null_print to file encoding,
* because it will be sent directly with CopySendString.
*/
- if (cstate->need_transcoding)
+ if (cstate->edata->need_transcoding)
cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
cstate->opts.null_print_len,
cstate->file_encoding);
@@ -401,14 +385,14 @@ SendCopyBegin(CopyToState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->edata->copy_dest = COPY_FRONTEND;
}
static void
SendCopyEnd(CopyToState cstate)
{
/* Shouldn't have any unsent data */
- Assert(cstate->fe_msgbuf->len == 0);
+ Assert(cstate->edata->fe_msgbuf->len == 0);
/* Send Copy Done message */
pq_putemptymessage(PqMsg_CopyDone);
}
@@ -426,32 +410,32 @@ SendCopyEnd(CopyToState cstate)
static void
CopySendData(CopyToState cstate, const void *databuf, int datasize)
{
- appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
+ appendBinaryStringInfo(cstate->edata->fe_msgbuf, databuf, datasize);
}
static void
CopySendString(CopyToState cstate, const char *str)
{
- appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
+ appendBinaryStringInfo(cstate->edata->fe_msgbuf, str, strlen(str));
}
static void
CopySendChar(CopyToState cstate, char c)
{
- appendStringInfoCharMacro(cstate->fe_msgbuf, c);
+ appendStringInfoCharMacro(cstate->edata->fe_msgbuf, c);
}
static void
CopySendEndOfRow(CopyToState cstate)
{
- StringInfo fe_msgbuf = cstate->fe_msgbuf;
+ StringInfo fe_msgbuf = cstate->edata->fe_msgbuf;
- switch (cstate->copy_dest)
+ switch (cstate->edata->copy_dest)
{
case COPY_FILE:
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
- cstate->copy_file) != 1 ||
- ferror(cstate->copy_file))
+ cstate->edata->copy_file) != 1 ||
+ ferror(cstate->edata->copy_file))
{
if (cstate->is_program)
{
@@ -492,8 +476,8 @@ CopySendEndOfRow(CopyToState cstate)
}
/* Update the progress */
- cstate->bytes_processed += fe_msgbuf->len;
- pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
+ cstate->edata->bytes_processed += fe_msgbuf->len;
+ pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->edata->bytes_processed);
resetStringInfo(fe_msgbuf);
}
@@ -505,7 +489,7 @@ CopySendEndOfRow(CopyToState cstate)
static inline void
CopySendTextLikeEndOfRow(CopyToState cstate)
{
- switch (cstate->copy_dest)
+ switch (cstate->edata->copy_dest)
{
case COPY_FILE:
/* Default line termination depends on platform */
@@ -565,7 +549,7 @@ ClosePipeToProgram(CopyToState cstate)
Assert(cstate->is_program);
- pclose_rc = ClosePipeStream(cstate->copy_file);
+ pclose_rc = ClosePipeStream(cstate->edata->copy_file);
if (pclose_rc == -1)
ereport(ERROR,
(errcode_for_file_access(),
@@ -592,7 +576,7 @@ EndCopy(CopyToState cstate)
}
else
{
- if (cstate->filename != NULL && FreeFile(cstate->copy_file))
+ if (cstate->filename != NULL && FreeFile(cstate->edata->copy_file))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not close file \"%s\": %m",
@@ -601,7 +585,7 @@ EndCopy(CopyToState cstate)
pgstat_progress_end_command();
- MemoryContextDelete(cstate->copycontext);
+ MemoryContextDelete(cstate->edata->copycontext);
pfree(cstate);
}
@@ -688,16 +672,17 @@ BeginCopyTo(ParseState *pstate,
/* Allocate workspace and zero all fields */
cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateData));
+ cstate->edata = (CopyToExecutionData *) palloc0(sizeof(CopyToExecutionData));
/*
* We allocate everything used by a cstate in a new memory context. This
* avoids memory leaks during repeated use of COPY in a query.
*/
- cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
- "COPY",
- ALLOCSET_DEFAULT_SIZES);
+ cstate->edata->copycontext = AllocSetContextCreate(CurrentMemoryContext,
+ "COPY",
+ ALLOCSET_DEFAULT_SIZES);
- oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ oldcontext = MemoryContextSwitchTo(cstate->edata->copycontext);
/* Extract options from the statement node tree */
ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options);
@@ -895,19 +880,19 @@ BeginCopyTo(ParseState *pstate,
*/
if (cstate->file_encoding == GetDatabaseEncoding() ||
cstate->file_encoding == PG_SQL_ASCII)
- cstate->need_transcoding = false;
+ cstate->edata->need_transcoding = false;
else
- cstate->need_transcoding = true;
+ cstate->edata->need_transcoding = true;
/* See Multibyte encoding comment above */
- cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
+ cstate->edata->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->edata->copy_dest = COPY_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->edata->copy_dest = COPY_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
@@ -916,7 +901,7 @@ BeginCopyTo(ParseState *pstate,
Assert(!is_program); /* the grammar does not allow this */
if (whereToSendOutput != DestRemote)
- cstate->copy_file = stdout;
+ cstate->edata->copy_file = stdout;
}
else
{
@@ -926,8 +911,8 @@ BeginCopyTo(ParseState *pstate,
if (is_program)
{
progress_vals[1] = PROGRESS_COPY_TYPE_PROGRAM;
- cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
- if (cstate->copy_file == NULL)
+ cstate->edata->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
+ if (cstate->edata->copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not execute command \"%s\": %m",
@@ -952,14 +937,14 @@ BeginCopyTo(ParseState *pstate,
oumask = umask(S_IWGRP | S_IWOTH);
PG_TRY();
{
- cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
+ cstate->edata->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
}
PG_FINALLY();
{
umask(oumask);
}
PG_END_TRY();
- if (cstate->copy_file == NULL)
+ if (cstate->edata->copy_file == NULL)
{
/* copy errno because ereport subfunctions might change it */
int save_errno = errno;
@@ -973,7 +958,7 @@ BeginCopyTo(ParseState *pstate,
"You may want a client-side facility such as psql's \\copy.") : 0));
}
- if (fstat(fileno(cstate->copy_file), &st))
+ if (fstat(fileno(cstate->edata->copy_file), &st))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m",
@@ -991,7 +976,7 @@ BeginCopyTo(ParseState *pstate,
cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid);
pgstat_progress_update_multi_param(2, progress_cols, progress_vals);
- cstate->bytes_processed = 0;
+ cstate->edata->bytes_processed = 0;
MemoryContextSwitchTo(oldcontext);
@@ -1042,8 +1027,8 @@ DoCopyTo(CopyToState cstate)
num_phys_attrs = tupDesc->natts;
cstate->opts.null_print_client = cstate->opts.null_print; /* default */
- /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
- cstate->fe_msgbuf = makeStringInfo();
+ /* We use fe_msgbuf as a per-row buffer regardless of edata->copy_dest */
+ cstate->edata->fe_msgbuf = makeStringInfo();
/* Get info about the columns we need to process. */
cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
@@ -1062,9 +1047,9 @@ DoCopyTo(CopyToState cstate)
* datatype output routines, and should be faster than retail pfree's
* anyway. (We don't need a whole econtext as CopyFrom does.)
*/
- cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
- "COPY TO",
- ALLOCSET_DEFAULT_SIZES);
+ cstate->edata->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
+ "COPY TO",
+ ALLOCSET_DEFAULT_SIZES);
cstate->routine->CopyToStart(cstate, tupDesc);
@@ -1107,7 +1092,7 @@ DoCopyTo(CopyToState cstate)
cstate->routine->CopyToEnd(cstate);
- MemoryContextDelete(cstate->rowcontext);
+ MemoryContextDelete(cstate->edata->rowcontext);
if (fe_copy)
SendCopyEnd(cstate);
@@ -1123,8 +1108,8 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
{
MemoryContext oldcontext;
- MemoryContextReset(cstate->rowcontext);
- oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
+ MemoryContextReset(cstate->edata->rowcontext);
+ oldcontext = MemoryContextSwitchTo(cstate->edata->rowcontext);
/* Make sure the tuple is fully deconstructed */
slot_getallattrs(slot);
@@ -1151,7 +1136,7 @@ CopyAttributeOutText(CopyToState cstate, const char *string)
char c;
char delimc = cstate->opts.delim[0];
- if (cstate->need_transcoding)
+ if (cstate->edata->need_transcoding)
ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
else
ptr = string;
@@ -1170,7 +1155,7 @@ CopyAttributeOutText(CopyToState cstate, const char *string)
* it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
* of the normal safe-encoding path.
*/
- if (cstate->encoding_embeds_ascii)
+ if (cstate->edata->encoding_embeds_ascii)
{
start = ptr;
while ((c = *ptr) != '\0')
@@ -1312,7 +1297,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
if (!use_quote && strcmp(string, cstate->opts.null_print) == 0)
use_quote = true;
- if (cstate->need_transcoding)
+ if (cstate->edata->need_transcoding)
ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
else
ptr = string;
@@ -1342,7 +1327,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
use_quote = true;
break;
}
- if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
+ if (IS_HIGHBIT_SET(c) && cstate->edata->encoding_embeds_ascii)
tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
else
tptr++;
@@ -1366,7 +1351,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
CopySendChar(cstate, escapec);
start = ptr; /* we include char in next run */
}
- if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
+ if (IS_HIGHBIT_SET(c) && cstate->edata->encoding_embeds_ascii)
ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
else
ptr++;
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 541176e1980..1fd44a132be 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -14,6 +14,7 @@
#ifndef COPY_H
#define COPY_H
+#include "executor/execdesc.h"
#include "nodes/execnodes.h"
#include "nodes/parsenodes.h"
#include "parser/parse_node.h"
@@ -87,13 +88,61 @@ typedef struct CopyFormatOptions
List *convert_select; /* list of column names (can be NIL) */
} CopyFormatOptions;
+typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
+typedef void (*copy_data_dest_cb) (void *data, int len);
+
+typedef struct CopyToStateData
+{
+ /* format-specific routines */
+ const struct CopyToRoutine *routine;
+
+ struct CopyToExecutionData *edata;
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
+
+ CopyFormatOptions opts;
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+} CopyToStateData;
+
+typedef struct CopyFromStateData
+{
+ /* format routine */
+ const struct CopyFromRoutine *routine;
+
+ struct CopyFromExecutionData *edata;
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ Oid conversion_proc; /* encoding conversion function */
+
+ CopyFormatOptions opts;
+
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+} CopyFromStateData;
+
/* These are private in commands/copy[from|to].c */
typedef struct CopyFromStateData *CopyFromState;
typedef struct CopyToStateData *CopyToState;
-typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
-typedef void (*copy_data_dest_cb) (void *data, int len);
-
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
uint64 *processed);
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index c8b22af22d8..b91b907e8eb 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -56,29 +56,17 @@ typedef enum CopyInsertMethod
* This struct contains all the state variables used throughout a COPY FROM
* operation.
*/
-typedef struct CopyFromStateData
+typedef struct CopyFromExecutionData
{
- /* format routine */
- const struct CopyFromRoutine *routine;
-
/* low-level state data */
CopySource copy_src; /* type of copy source */
FILE *copy_file; /* used if copy_src == COPY_FILE */
StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
EolType eol_type; /* EOL type of input */
- int file_encoding; /* file or remote side's character encoding */
bool need_transcoding; /* file encoding diff from server? */
Oid conversion_proc; /* encoding conversion function */
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
Node *whereClause; /* WHERE condition (or NULL) */
@@ -96,8 +84,6 @@ typedef struct CopyFromStateData
AttrNumber num_defaults; /* count of att that are missing and have
* default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
ErrorSaveContext *escontext; /* soft error trapped during in_functions
* execution */
uint64 num_errors; /* total number of rows which contained soft
@@ -164,7 +150,7 @@ typedef struct CopyFromStateData
bool input_reached_eof; /* true if we reached EOF */
bool input_reached_error; /* true if a conversion error happened */
/* Shorthand for number of unconsumed bytes available in input_buf */
-#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
+#define INPUT_BUF_BYTES(edata) ((edata)->input_buf_len - (edata)->input_buf_index)
/*
* raw_buf holds raw input data read from the data source (file or client
@@ -178,10 +164,10 @@ typedef struct CopyFromStateData
bool raw_reached_eof; /* true if we reached EOF */
/* Shorthand for number of unconsumed bytes available in raw_buf */
-#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
+#define RAW_BUF_BYTES(edata) ((edata)->raw_buf_len - (edata)->raw_buf_index)
uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
+} CopyFromExecutionData;
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
--
2.43.5
[application/octet-stream] 0002-Custom-COPY-format.patch (18.9K, ../../CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com/3-0002-Custom-COPY-format.patch)
download | inline diff:
From 322512cbcf4b2e6c07e6bd8f9d4b2afc2c400193 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Sun, 13 Jul 2025 10:09:57 -0700
Subject: [PATCH 2/2] Custom COPY format.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
src/backend/commands/Makefile | 1 +
src/backend/commands/copy.c | 73 +++++++++------
src/backend/commands/copyformat.c | 134 +++++++++++++++++++++++++++
src/backend/commands/copyfrom.c | 26 ++++--
src/backend/commands/copyfromparse.c | 7 +-
src/backend/commands/copyto.c | 28 ++++--
src/include/commands/copy.h | 18 +++-
src/include/commands/copyapi.h | 5 +
8 files changed, 244 insertions(+), 48 deletions(-)
create mode 100644 src/backend/commands/copyformat.c
diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile
index cb2fbdc7c60..30693ffba27 100644
--- a/src/backend/commands/Makefile
+++ b/src/backend/commands/Makefile
@@ -24,6 +24,7 @@ OBJS = \
constraint.o \
conversioncmds.o \
copy.o \
+ copyformat.o \
copyfrom.o \
copyfromparse.o \
copyto.o \
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index fae9c41db65..4e482955544 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -514,6 +514,8 @@ ProcessCopyOptions(ParseState *pstate,
bool on_error_specified = false;
bool log_verbosity_specified = false;
bool reject_limit_specified = false;
+ bool csv_mode = false;
+ bool binary = false;
ListCell *option;
/* Support external use for option sanity checking */
@@ -521,6 +523,8 @@ ProcessCopyOptions(ParseState *pstate,
opts_out = (CopyFormatOptions *) palloc0(sizeof(CopyFormatOptions));
opts_out->file_encoding = -1;
+ opts_out->format = COPY_FORMAT_TEXT;
+ opts_out->format_ext_id = -1;
/* Extract options from the statement node tree */
foreach(option, options)
@@ -535,16 +539,29 @@ ProcessCopyOptions(ParseState *pstate,
errorConflictingDefElem(defel, pstate);
format_specified = true;
if (strcmp(fmt, "text") == 0)
- /* default format */ ;
+ opts_out->format = COPY_FORMAT_TEXT;
else if (strcmp(fmt, "csv") == 0)
- opts_out->csv_mode = true;
+ {
+ opts_out->format = COPY_FORMAT_CSV;
+ csv_mode = true;
+ }
else if (strcmp(fmt, "binary") == 0)
- opts_out->binary = true;
+ {
+ opts_out->format = COPY_FORMAT_BINARY;
+ binary = true;
+ }
else
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format \"%s\" not recognized", fmt),
- parser_errposition(pstate, defel->location)));
+ {
+ opts_out->format_ext_id = GetCopyFormatExtensionId(defel->defname);
+
+ if (opts_out->format_ext_id == -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", fmt),
+ parser_errposition(pstate, defel->location)));
+
+ opts_out->format = COPY_FORMAT_EXTENSION;
+ }
}
else if (strcmp(defel->defname, "freeze") == 0)
{
@@ -699,31 +716,31 @@ ProcessCopyOptions(ParseState *pstate,
* Check for incompatible options (must do these three before inserting
* defaults)
*/
- if (opts_out->binary && opts_out->delim)
+ if (binary && opts_out->delim)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
errmsg("cannot specify %s in BINARY mode", "DELIMITER")));
- if (opts_out->binary && opts_out->null_print)
+ if (binary && opts_out->null_print)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("cannot specify %s in BINARY mode", "NULL")));
- if (opts_out->binary && opts_out->default_print)
+ if (binary && opts_out->default_print)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("cannot specify %s in BINARY mode", "DEFAULT")));
/* Set defaults for omitted options */
if (!opts_out->delim)
- opts_out->delim = opts_out->csv_mode ? "," : "\t";
+ opts_out->delim = csv_mode ? "," : "\t";
if (!opts_out->null_print)
- opts_out->null_print = opts_out->csv_mode ? "" : "\\N";
+ opts_out->null_print = csv_mode ? "" : "\\N";
opts_out->null_print_len = strlen(opts_out->null_print);
- if (opts_out->csv_mode)
+ if (csv_mode)
{
if (!opts_out->quote)
opts_out->quote = "\"";
@@ -771,7 +788,7 @@ ProcessCopyOptions(ParseState *pstate,
* future-proofing. Likewise we disallow all digits though only octal
* digits are actually dangerous.
*/
- if (!opts_out->csv_mode &&
+ if (!csv_mode &&
strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
opts_out->delim[0]) != NULL)
ereport(ERROR,
@@ -779,43 +796,43 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim)));
/* Check header */
- if (opts_out->binary && opts_out->header_line != COPY_HEADER_FALSE)
+ if (binary && opts_out->header_line != COPY_HEADER_FALSE)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
errmsg("cannot specify %s in BINARY mode", "HEADER")));
/* Check quote */
- if (!opts_out->csv_mode && opts_out->quote != NULL)
+ if (!csv_mode && opts_out->quote != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
errmsg("COPY %s requires CSV mode", "QUOTE")));
- if (opts_out->csv_mode && strlen(opts_out->quote) != 1)
+ if (csv_mode && strlen(opts_out->quote) != 1)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY quote must be a single one-byte character")));
- if (opts_out->csv_mode && opts_out->delim[0] == opts_out->quote[0])
+ if (csv_mode && opts_out->delim[0] == opts_out->quote[0])
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY delimiter and quote must be different")));
/* Check escape */
- if (!opts_out->csv_mode && opts_out->escape != NULL)
+ if (!csv_mode && opts_out->escape != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
errmsg("COPY %s requires CSV mode", "ESCAPE")));
- if (opts_out->csv_mode && strlen(opts_out->escape) != 1)
+ if (csv_mode && strlen(opts_out->escape) != 1)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY escape must be a single one-byte character")));
/* Check force_quote */
- if (!opts_out->csv_mode && (opts_out->force_quote || opts_out->force_quote_all))
+ if (!csv_mode && (opts_out->force_quote || opts_out->force_quote_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -829,8 +846,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY FROM")));
/* Check force_notnull */
- if (!opts_out->csv_mode && (opts_out->force_notnull != NIL ||
- opts_out->force_notnull_all))
+ if (!csv_mode && (opts_out->force_notnull != NIL ||
+ opts_out->force_notnull_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -845,8 +862,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY TO")));
/* Check force_null */
- if (!opts_out->csv_mode && (opts_out->force_null != NIL ||
- opts_out->force_null_all))
+ if (!csv_mode && (opts_out->force_null != NIL ||
+ opts_out->force_null_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -870,7 +887,7 @@ ProcessCopyOptions(ParseState *pstate,
"NULL")));
/* Don't allow the CSV quote char to appear in the null string. */
- if (opts_out->csv_mode &&
+ if (csv_mode &&
strchr(opts_out->null_print, opts_out->quote[0]) != NULL)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -906,7 +923,7 @@ ProcessCopyOptions(ParseState *pstate,
"DEFAULT")));
/* Don't allow the CSV quote char to appear in the default string. */
- if (opts_out->csv_mode &&
+ if (csv_mode &&
strchr(opts_out->default_print, opts_out->quote[0]) != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -923,7 +940,7 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("NULL specification and DEFAULT specification cannot be the same")));
}
/* Check on_error */
- if (opts_out->binary && opts_out->on_error != COPY_ON_ERROR_STOP)
+ if (binary && opts_out->on_error != COPY_ON_ERROR_STOP)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("only ON_ERROR STOP is allowed in BINARY mode")));
diff --git a/src/backend/commands/copyformat.c b/src/backend/commands/copyformat.c
new file mode 100644
index 00000000000..6d1c40833e9
--- /dev/null
+++ b/src/backend/commands/copyformat.c
@@ -0,0 +1,134 @@
+/*-------------------------------------------------------------------------
+ *
+ * copyformat.c
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/commands/copyformat.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "commands/copyapi.h"
+#include "utils/memutils.h"
+
+typedef struct CopyFormatExtensionInfo
+{
+ const char *format_name;
+ const CopyToRoutine *to_routine;
+ const CopyFromRoutine *from_routine;
+} CopyFormatExtensionInfo;
+
+static CopyFormatExtensionInfo * CopyFormatExtensionArray = NULL;
+static int CopyFormatExtensionAssigned = 0;
+static int CopyFormatExtensionAllocated = 0;
+
+/*
+ * Register new COPY routines for the given name.
+ *
+ * When name is used as a COPY format name, routine will be used to process
+ * the COPY request. See CopyFromRoutine and CopyToRoutine how to implement a
+ * COPY routine.
+ *
+ * If name is already registered, an error is raised.
+ *
+ * name is assumed to be a constant string or allocated in storage that will
+ * never be freed.
+ *
+ * fmt, from_routine, and to_routine are assumed to be allocated in storage
+ * that will never be freed.
+ */
+void
+RegisterCustomCopyFormat(const char *fmt, const CopyToRoutine *to_routine,
+ const CopyFromRoutine *from_routine)
+{
+ CopyFormatExtensionInfo *info;
+
+ for (int i = 0; i < CopyFormatExtensionAssigned; i++)
+ {
+ if (strcmp(fmt, CopyFormatExtensionArray[i].format_name) == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_OBJECT),
+ errmsg("custom COPY fromat \"%s\" already exists",
+ fmt)));
+ }
+
+ /* If there is no array yet, create one */
+ if (CopyFormatExtensionAllocated == 0)
+ {
+ CopyFormatExtensionAllocated = 16;
+ CopyFormatExtensionArray = (CopyFormatExtensionInfo *)
+ MemoryContextAlloc(TopMemoryContext,
+ CopyFormatExtensionAllocated * sizeof(CopyFormatExtensionInfo));
+ }
+
+ /* If there's an array but it's currently full, expand it */
+ if (CopyFormatExtensionAssigned >= CopyFormatExtensionAllocated)
+ {
+ int i = pg_nextpower2_32(CopyFormatExtensionAssigned + 1);
+
+ CopyFormatExtensionArray =
+ repalloc(CopyFormatExtensionArray, sizeof(CopyFormatExtensionInfo) * i);
+ CopyFormatExtensionAllocated = i;
+ }
+
+ /* Register the format */
+ info = &CopyFormatExtensionArray[CopyFormatExtensionAssigned++];
+ info->format_name = fmt;
+ info->from_routine = from_routine;
+ info->to_routine = to_routine;
+}
+
+/*
+ * Get an integer ID of the given name of an custom COPY format extension.
+ *
+ * Within the lifetime of a particular backend, the same name will be
+ * mapped to the same ID every time. IDs of COPY formats that are not
+ * loaded during shared_preload_libraries time are not stable across
+ * backends.
+ */
+int
+GetCopyFormatExtensionId(const char *fmt)
+{
+ for (int i = 0; i < CopyFormatExtensionAssigned; i++)
+ {
+ if (strcmp(CopyFormatExtensionArray[i].format_name, fmt) == 0)
+ return i;
+ }
+
+ return -1;
+}
+
+const CopyToRoutine *
+GetCustomCopyToRoutine(int extension_id)
+{
+ if (extension_id >= CopyFormatExtensionAssigned || extension_id < 0)
+ return NULL;
+
+ if (!CopyFormatExtensionArray[extension_id].to_routine)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("COPY format \"%s\" does not support COPY TO routine",
+ CopyFormatExtensionArray[extension_id].format_name)));
+
+ return CopyFormatExtensionArray[extension_id].to_routine;
+}
+
+const CopyFromRoutine *
+GetCustomCopyFromRoutine(int extension_id)
+{
+ if (extension_id >= CopyFormatExtensionAssigned || extension_id < 0)
+ return NULL;
+
+ if (!CopyFormatExtensionArray[extension_id].to_routine)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("COPY format \"%s\" does not support COPY FROM routine",
+ CopyFormatExtensionArray[extension_id].format_name)));
+
+ return CopyFormatExtensionArray[extension_id].from_routine;
+}
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 03779838654..fc38caaca24 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -155,13 +155,25 @@ static const CopyFromRoutine CopyFromRoutineBinary = {
static const CopyFromRoutine *
CopyFromGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->csv_mode)
- return &CopyFromRoutineCSV;
- else if (opts->binary)
- return &CopyFromRoutineBinary;
+ const CopyFromRoutine *routine;
- /* default is text */
- return &CopyFromRoutineText;
+ switch (opts->format)
+ {
+ case COPY_FORMAT_CSV:
+ routine = &CopyFromRoutineCSV;
+ break;
+ case COPY_FORMAT_TEXT:
+ routine = &CopyFromRoutineText;
+ break;
+ case COPY_FORMAT_BINARY:
+ routine = &CopyFromRoutineBinary;
+ break;
+ case COPY_FORMAT_EXTENSION:
+ routine = GetCustomCopyFromRoutine(opts->format_ext_id);
+ break;
+ }
+
+ return routine;
}
/* Implementation of the start callback for text and CSV formats */
@@ -263,7 +275,7 @@ CopyFromErrorCallback(void *arg)
edata->cur_relname);
return;
}
- if (cstate->opts.binary)
+ if (cstate->opts.format == COPY_FORMAT_BINARY)
{
/* can't usefully display the data */
if (edata->cur_attname)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 595dc84b172..0b94d8851f3 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -171,7 +171,7 @@ ReceiveCopyBegin(CopyFromState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0);
int i;
pq_beginmessage(&buf, PqMsg_CopyInResponse);
@@ -754,7 +754,7 @@ bool
NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
{
return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
- cstate->opts.csv_mode);
+ cstate->opts.format == COPY_FORMAT_CSV);
}
/*
@@ -782,7 +782,8 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
bool done = false;
/* only available for text or csv input */
- Assert(!cstate->opts.binary);
+ Assert(cstate->opts.format == COPY_FORMAT_TEXT ||
+ cstate->opts.format == COPY_FORMAT_CSV);
/* on input check that the header line is correct if needed */
if (edata->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index e90fab6d958..842c5efa8d5 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -160,13 +160,25 @@ static const CopyToRoutine CopyToRoutineBinary = {
static const CopyToRoutine *
CopyToGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->csv_mode)
- return &CopyToRoutineCSV;
- else if (opts->binary)
- return &CopyToRoutineBinary;
+ const CopyToRoutine *routine;
- /* default is text */
- return &CopyToRoutineText;
+ switch (opts->format)
+ {
+ case COPY_FORMAT_CSV:
+ routine = &CopyToRoutineCSV;
+ break;
+ case COPY_FORMAT_TEXT:
+ routine = &CopyToRoutineText;
+ break;
+ case COPY_FORMAT_BINARY:
+ routine = &CopyToRoutineBinary;
+ break;
+ case COPY_FORMAT_EXTENSION:
+ routine = GetCustomCopyToRoutine(opts->format_ext_id);
+ break;
+ }
+
+ return routine;
}
/* Implementation of the start callback for text and CSV formats */
@@ -199,7 +211,7 @@ CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc)
colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
- if (cstate->opts.csv_mode)
+ if (cstate->opts.format == COPY_FORMAT_CSV)
CopyAttributeOutCSV(cstate, colname, false);
else
CopyAttributeOutText(cstate, colname);
@@ -376,7 +388,7 @@ SendCopyBegin(CopyToState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0);
int i;
pq_beginmessage(&buf, PqMsg_CopyOutResponse);
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 1fd44a132be..73660e694ce 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -49,6 +49,17 @@ typedef enum CopyLogVerbosityChoice
COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */
} CopyLogVerbosityChoice;
+/*
+ * Represents the file format processed by COPY command.
+ */
+typedef enum CopyFormatType
+{
+ COPY_FORMAT_CSV = 0,
+ COPY_FORMAT_TEXT,
+ COPY_FORMAT_BINARY,
+ COPY_FORMAT_EXTENSION,
+} CopyFormatType;
+
/*
* A struct to hold COPY options, in a parsed form. All of these are related
* to formatting, except for 'freeze', which doesn't really belong here, but
@@ -59,9 +70,10 @@ typedef struct CopyFormatOptions
/* parameters from the COPY command */
int file_encoding; /* file or remote side's character encoding,
* -1 if not specified */
- bool binary; /* binary format? */
+ CopyFormatType format;
+ int format_ext_id; /* custom COPY format id, -1 if format is
+ * built-in format */
bool freeze; /* freeze rows on loading? */
- bool csv_mode; /* Comma Separated Value format? */
int header_line; /* number of lines to skip or COPY_HEADER_XXX
* value (see the above) */
char *null_print; /* NULL marker string (server encoding!) */
@@ -163,6 +175,8 @@ extern uint64 CopyFrom(CopyFromState cstate);
extern DestReceiver *CreateCopyDestReceiver(void);
+extern int GetCopyFormatExtensionId(const char *fmt);
+
/*
* internal prototypes
*/
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 2a2d2f9876b..b9e29dddd5b 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -102,4 +102,9 @@ typedef struct CopyFromRoutine
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
+extern void RegisterCustomCopyFormat(const char *fmt, const CopyToRoutine *to_routine,
+ const CopyFromRoutine *from_routine);
+extern const CopyToRoutine *GetCustomCopyToRoutine(int extension_id);
+extern const CopyFromRoutine *GetCustomCopyFromRoutine(int extension_id);
+
#endif /* COPYAPI_H */
--
2.43.5
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-14 08:38 ` Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-07-14 08:38 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 03:28:16 +0900,
Masahiko Sawada <[email protected]> wrote:
> I've reviewed the 0001 and 0002 patches. The API implemented in the
> 0002 patch looks good to me, but I'm concerned about the capsulation
> of copy state data. With the v42 patches, we pass the whole
> CopyToStateData to the extension codes, but most of the fields in
> CopyToStateData are internal working state data that shouldn't be
> exposed to extensions. I think we need to sort out which fields are
> exposed or not. That way, it would be safer and we would be able to
> avoid exposing copyto_internal.h and extensions would not need to
> include copyfrom_internal.h.
FYI: We discussed this so far. For example:
https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m...
> I think we can move CopyToState to copy.h and we don't
> need to have set/get functions for its fields.
https://www.postgresql.org/message-id/flat/CAD21AoBpWFU4k-_bwrTq0AkFSAdwQqhAsSW188STmu9HxLJ0nQ%40mai...
> > What does "private" mean here? I thought that it means that
> > "PostgreSQL itself can use it". But it seems that you mean
> > that "PostgreSQL itself and custom format extensions can use
> > it but other extensions can't use it".
> >
> > I'm not familiar with "_internal.h" in PostgreSQL but is
> > "_internal.h" for the latter "private" mean?
>
> My understanding is that we don't strictly prohibit _internal.h from
> being included in out of core files. For example, file_fdw.c includes
> copyfrom_internal.h in order to access some fields of CopyFromState.
In general, I agree that we should export only needed
information.
How about adding accessors instead of splitting
Copy{From,To}State to Copy{From,To}ExecutionData? If we use
the accessors approach, we can export only needed
information step by step without breaking ABI.
The built-in formats can keep using Copy{From,To}State
directly with the accessors approach. We can avoid any
performance regression of the built-in formats. If we split
Copy{From,To}State to Copy{From,To}ExecutionData,
performance may be changed.
> I've implemented a draft patch for that idea. In the 0001 patch, I
> moved fields that are related to internal working state from
> CopyToStateData to CopyToExectuionData. COPY routine APIs pass a
> pointer of CopyToStateData but extensions can access only fields
> except for CopyToExectuionData. In the 0002 patch, I've implemented
> the registration API and some related APIs based on your v42 patch.
> I've made similar changes to COPY FROM codes too.
>
> The patch is a very PoC phase and we would need to scrutinize the
> fields that should or should not be exposed. Feedback is very welcome.
Based on our sample extensions [1][2], the following fields
may be minimal. I added "(*)" marks that exist in
Copy{From,To}StateDate in your patch. Other fields exist in
Copy{From,To}ExecutionData. We need to export them to
extensions. We can hide fields in Copy{From,To}StateData not
listed here.
FROM:
- attnumlist (*)
- bytes_processed
- cur_attname
- escontext
- in_functions (*)
- input_buf
- input_reached_eof
- line_buf
- opts (*)
- raw_buf
- raw_buf_index
- raw_buf_len
- rel (*)
- typioparams (*)
TO:
- attnumlist (*)
- fe_msgbuf
- opts (*)
[1] https://github.com/kou/pg-copy-arrow/
[2] https://github.com/MasahikoSawada/pg_copy_jsonlines/
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-07-15 07:54 ` Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-07-15 07:54 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 17:38:03 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
>> I've reviewed the 0001 and 0002 patches. The API implemented in the
>> 0002 patch looks good to me, but I'm concerned about the capsulation
>> of copy state data. With the v42 patches, we pass the whole
>> CopyToStateData to the extension codes, but most of the fields in
>> CopyToStateData are internal working state data that shouldn't be
>> exposed to extensions. I think we need to sort out which fields are
>> exposed or not. That way, it would be safer and we would be able to
>> avoid exposing copyto_internal.h and extensions would not need to
>> include copyfrom_internal.h.
> In general, I agree that we should export only needed
> information.
>
> How about adding accessors instead of splitting
> Copy{From,To}State to Copy{From,To}ExecutionData? If we use
> the accessors approach, we can export only needed
> information step by step without breaking ABI.
Another idea: We'll add Copy{From,To}State::opaque
eventually. (For example, the v40-0003 patch includes it.)
How about using it to hide fields only for built-in formats?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-07-17 20:44 ` Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-07-17 20:44 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jul 15, 2025 at 12:54 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 17:38:03 +0900 (JST),
> Sutou Kouhei <[email protected]> wrote:
>
> >> I've reviewed the 0001 and 0002 patches. The API implemented in the
> >> 0002 patch looks good to me, but I'm concerned about the capsulation
> >> of copy state data. With the v42 patches, we pass the whole
> >> CopyToStateData to the extension codes, but most of the fields in
> >> CopyToStateData are internal working state data that shouldn't be
> >> exposed to extensions. I think we need to sort out which fields are
> >> exposed or not. That way, it would be safer and we would be able to
> >> avoid exposing copyto_internal.h and extensions would not need to
> >> include copyfrom_internal.h.
>
> > In general, I agree that we should export only needed
> > information.
> >
> > How about adding accessors instead of splitting
> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use
> > the accessors approach, we can export only needed
> > information step by step without breaking ABI.
Yeah, while it can export required fields without breaking ABI, I'm
concerned that setter and getter functions could be bloated if we need
to have them for many fields.
>
> Another idea: We'll add Copy{From,To}State::opaque
> eventually. (For example, the v40-0003 patch includes it.)
>
> How about using it to hide fields only for built-in formats?
What is the difference between your idea and splitting CopyToState
into CopyToState and CopyToExecutionData?
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-18 10:05 ` Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-07-18 10:05 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700,
Masahiko Sawada <[email protected]> wrote:
>> > How about adding accessors instead of splitting
>> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use
>> > the accessors approach, we can export only needed
>> > information step by step without breaking ABI.
>
> Yeah, while it can export required fields without breaking ABI, I'm
> concerned that setter and getter functions could be bloated if we need
> to have them for many fields.
In general, I choose this approach in my projects even when
I need to define many accessors. Because I can hide
implementation details from users. I can change
implementation details without breaking API/ABI.
But PostgreSQL isn't my project. Is there any guideline for
PostgreSQL API(/ABI?) design that we can refer for this
case?
FYI: We need to export at least the following fields:
https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7...
> FROM:
>
> - attnumlist (*)
> - bytes_processed
> - cur_attname
> - escontext
> - in_functions (*)
> - input_buf
> - input_reached_eof
> - line_buf
> - opts (*)
> - raw_buf
> - raw_buf_index
> - raw_buf_len
> - rel (*)
> - typioparams (*)
>
> TO:
>
> - attnumlist (*)
> - fe_msgbuf
> - opts (*)
Here are pros/cons of the Copy{From,To}ExecutionData
approach, right?
Pros:
1. We can hide internal data from extensions
Cons:
1. Built-in format routines need to refer fields via
Copy{From,To}ExecutionData.
* This MAY has performance impact. If there is no
performance impact, this is not a cons.
2. API/ABI compatibility will be broken when we change
exported fields.
* I'm not sure whether this is a cons in the PostgreSQL
design.
Here are pros/cons of the accessors approach:
Pros:
1. We can hide internal data from extensions
2. We can export new fields change field names
without breaking API/ABI compatibility
3. We don't need to change built-in format routines.
So we can assume that there is no performance impact.
Cons:
1. We may need to define many accessors
* I'm not sure whether this is a cons in the PostgreSQL
design.
>> Another idea: We'll add Copy{From,To}State::opaque
>> eventually. (For example, the v40-0003 patch includes it.)
>>
>> How about using it to hide fields only for built-in formats?
>
> What is the difference between your idea and splitting CopyToState
> into CopyToState and CopyToExecutionData?
1. We don't need to manage 2 similar data for built-in
formats and extensions.
* Build-in formats use CopyToExecutionData and extensions
use opaque.
2. We can introduce registration API now.
* We can work on this topic AFTER we introduce
registration API.
* e.g.: Add registration API -> Add opaque -> Use opaque
for internal fields (we will benchmark this
implementation at this time)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-07-28 19:33 ` Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-07-28 19:33 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jul 18, 2025 at 3:05 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> > How about adding accessors instead of splitting
> >> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use
> >> > the accessors approach, we can export only needed
> >> > information step by step without breaking ABI.
> >
> > Yeah, while it can export required fields without breaking ABI, I'm
> > concerned that setter and getter functions could be bloated if we need
> > to have them for many fields.
>
> In general, I choose this approach in my projects even when
> I need to define many accessors. Because I can hide
> implementation details from users. I can change
> implementation details without breaking API/ABI.
>
> But PostgreSQL isn't my project. Is there any guideline for
> PostgreSQL API(/ABI?) design that we can refer for this
> case?
As far as I know there is no official guideline for PostgreSQL API and
ABI design, but I've never seen structs having more than 10 getter and
setter in PostgreSQL source code.
>
> FYI: We need to export at least the following fields:
>
> https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7...
>
> > FROM:
> >
> > - attnumlist (*)
> > - bytes_processed
> > - cur_attname
> > - escontext
> > - in_functions (*)
> > - input_buf
> > - input_reached_eof
> > - line_buf
> > - opts (*)
> > - raw_buf
> > - raw_buf_index
> > - raw_buf_len
> > - rel (*)
> > - typioparams (*)
> >
> > TO:
> >
> > - attnumlist (*)
> > - fe_msgbuf
> > - opts (*)
I think we can think of the minimum list of fields that we need to
expose. For instance, fields used for buffered reads for COPY FROM
such as input_buf and raw_buf related fields don't necessarily need to
be exposed as extension can implement it in its own way. We can start
with the supporting simple copy format extensions like that read and
parse the binary data from the data source and fill 'values' and
'nulls' arrays as output. Considering these facts, it might be
sufficient for copy format extensions if they could access 'rel',
'attnumlist', and 'opts' in both COPY FROM and COPY TO (and
CopyFromErrorCallback related fields for COPY FROM).
Apart from this, we might want to reorganize CopyFromStateData fields
and CopyToStateData fields since they have mixed fields of general
purpose fields for COPY operations (e.g., num_defaults, whereClause,
and range_table) and built-in format specific fields (e.g., line_buf
and input_buf). Text and CSV formats are using some fields for parsing
fields with buffered reads so one idea is that we move related fields
to another struct so that both built-in formats (text and CSV) and
external extensions that want to use the buffered reads for text
parsing can use this functionality.
> Here are pros/cons of the Copy{From,To}ExecutionData
> approach, right?
>
> Pros:
> 1. We can hide internal data from extensions
>
> Cons:
> 1. Built-in format routines need to refer fields via
> Copy{From,To}ExecutionData.
> * This MAY has performance impact. If there is no
> performance impact, this is not a cons.
> 2. API/ABI compatibility will be broken when we change
> exported fields.
> * I'm not sure whether this is a cons in the PostgreSQL
> design.
>
> Here are pros/cons of the accessors approach:
>
> Pros:
> 1. We can hide internal data from extensions
> 2. We can export new fields change field names
> without breaking API/ABI compatibility
> 3. We don't need to change built-in format routines.
> So we can assume that there is no performance impact.
>
> Cons:
> 1. We may need to define many accessors
> * I'm not sure whether this is a cons in the PostgreSQL
> design.
I agree with the summary.
> >> Another idea: We'll add Copy{From,To}State::opaque
> >> eventually. (For example, the v40-0003 patch includes it.)
> >>
> >> How about using it to hide fields only for built-in formats?
> >
> > What is the difference between your idea and splitting CopyToState
> > into CopyToState and CopyToExecutionData?
>
> 1. We don't need to manage 2 similar data for built-in
> formats and extensions.
> * Build-in formats use CopyToExecutionData and extensions
> use opaque.
> 2. We can introduce registration API now.
> * We can work on this topic AFTER we introduce
> registration API.
> * e.g.: Add registration API -> Add opaque -> Use opaque
> for internal fields (we will benchmark this
> implementation at this time)
What if we find performance overhead in built-in format cases after
introducing opaque data? I personally would like to avoid merging the
registration API (i.e., supporting custom copy formats) while being
unsure about the overall design ahead and the potential performance
impact by following patches.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-29 05:19 ` Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-07-29 05:19 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jul 28, 2025 at 12:33 PM Masahiko Sawada <[email protected]> wrote:
>
> On Fri, Jul 18, 2025 at 3:05 AM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700,
> > Masahiko Sawada <[email protected]> wrote:
> >
> > >> > How about adding accessors instead of splitting
> > >> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use
> > >> > the accessors approach, we can export only needed
> > >> > information step by step without breaking ABI.
> > >
> > > Yeah, while it can export required fields without breaking ABI, I'm
> > > concerned that setter and getter functions could be bloated if we need
> > > to have them for many fields.
> >
> > In general, I choose this approach in my projects even when
> > I need to define many accessors. Because I can hide
> > implementation details from users. I can change
> > implementation details without breaking API/ABI.
> >
> > But PostgreSQL isn't my project. Is there any guideline for
> > PostgreSQL API(/ABI?) design that we can refer for this
> > case?
>
> As far as I know there is no official guideline for PostgreSQL API and
> ABI design, but I've never seen structs having more than 10 getter and
> setter in PostgreSQL source code.
>
> >
> > FYI: We need to export at least the following fields:
> >
> > https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7...
> >
> > > FROM:
> > >
> > > - attnumlist (*)
> > > - bytes_processed
> > > - cur_attname
> > > - escontext
> > > - in_functions (*)
> > > - input_buf
> > > - input_reached_eof
> > > - line_buf
> > > - opts (*)
> > > - raw_buf
> > > - raw_buf_index
> > > - raw_buf_len
> > > - rel (*)
> > > - typioparams (*)
> > >
> > > TO:
> > >
> > > - attnumlist (*)
> > > - fe_msgbuf
> > > - opts (*)
>
> I think we can think of the minimum list of fields that we need to
> expose. For instance, fields used for buffered reads for COPY FROM
> such as input_buf and raw_buf related fields don't necessarily need to
> be exposed as extension can implement it in its own way. We can start
> with the supporting simple copy format extensions like that read and
> parse the binary data from the data source and fill 'values' and
> 'nulls' arrays as output. Considering these facts, it might be
> sufficient for copy format extensions if they could access 'rel',
> 'attnumlist', and 'opts' in both COPY FROM and COPY TO (and
> CopyFromErrorCallback related fields for COPY FROM).
>
> Apart from this, we might want to reorganize CopyFromStateData fields
> and CopyToStateData fields since they have mixed fields of general
> purpose fields for COPY operations (e.g., num_defaults, whereClause,
> and range_table) and built-in format specific fields (e.g., line_buf
> and input_buf). Text and CSV formats are using some fields for parsing
> fields with buffered reads so one idea is that we move related fields
> to another struct so that both built-in formats (text and CSV) and
> external extensions that want to use the buffered reads for text
> parsing can use this functionality.
So probably it might be worth refactoring the codes in terms of:
1. hiding internal data from format callbacks
2. separating format-specific fields from the main state data.
I categorized the fields in CopyFromStateData. I think there are
roughly three different kind of fields mixed there:
1. fields used only the core (not by format callback)
- filename
- is_program
- whereClause
- cur_relname
- copycontext
- defmap
- num_defaults
- volatile_defexprs
- range_table
- rtrperminfos
- qualexpr
- transition_capture
2. fields used by both the core and format callbacks
- rel
- attnumlist
- cur_lineno
- cur_attname
- cur_attval
- relname_only
- num_errors
- opts
- in_functions
- typioparams
- escontext
- defexprs
- Input-related fields
- copy_src
- copy_file
- fe_msgbuf
- data_source_cb
- byteprocessed
3. built-in format specific fields (mostly for text and csv)
- eol_type
- defaults
- Encoding related fields
- file_encoding
- need_transcoding
- conversion_proc
- convert_select_flags
- raw data pointers
- max_fields
- raw_fields
- attribute_buf
- line_buf related fields
- line_buf
- line_buf_valid
- input_buf related fields
- input_buf
- input_buf_index
- input_buf_len
- input_reached_eof
- input_reached_error
- raw_buf related fields
- raw_buf
- raw_buf_index
- raw_buf_len
- raw_reached_eof
The fields in 1 are mostly static fields, and the fields in 2 and 3
are likely to be accessed in hot functions during COPY FROM. Would it
be a good idea to restructure these fields so that we can hide the
fields in 1 from callback functions and having the fields in 3 in a
separate format-specific struct that can be accessed via an opaque
pointer? But could the latter change potentially cause performance
overheads?
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-08-14 06:36 ` Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-08-14 06:36 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC+gf=3w7XiA4LQnvx0g@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 28 Jul 2025 22:19:36 -0700,
Masahiko Sawada <[email protected]> wrote:
> The fields in 1 are mostly static fields, and the fields in 2 and 3
> are likely to be accessed in hot functions during COPY FROM. Would it
> be a good idea to restructure these fields so that we can hide the
> fields in 1 from callback functions and having the fields in 3 in a
> separate format-specific struct that can be accessed via an opaque
> pointer? But could the latter change potentially cause performance
> overheads?
Yes. It changes memory layout (1 continuous memory chunk ->
2 separated memory chunks) and introduces indirect member
accesses (x->y -> x->z->y). They may not have performance
impact but we need to measure it if we want to use this
approach.
BTW, how about the following approach?
copyapi.h:
typedef struct CopyToStateData
{
/* public members */
/* ... */
} CopyToStateData;
copyto.c:
typedef struct CopyToStateInternalData
{
CopyToStateData parent;
/* private members */
/* ... */
} CopyToStateInternalData;
We export CopyToStateData only with public members. We don't
export CopyToStateInternalData that has members only for
built-in formats.
CopyToStateInternalData has the same memory layout as
CopyToStateData. So we can use CopyToStateInternalData as
CopyToStateData.
We use CopyToStateData not CopyToStateInternalData in public
API. We cast CopyToStateData to CopyToStateInternalData when
we need to use private members:
static void
CopySendData(CopyToState cstate, const void *databuf, int datasize)
{
CopyToStateInternal cstate_internal = (CopyToStateInternal) cstate;
appendBinaryStringInfo(cstate_internal->fe_msgbuf, databuf, datasize);
}
It's still direct member access.
With this approach, we can keep the same memory layout (1
continuous memory chunk) and direct member access. I think
that this approach doesn't have performance impact.
See the attached patch for PoC of this approach.
Drawback: This approach always allocates
CopyToStateInternalData not CopyToStateData. So we need to
allocate needless memories for extensions. But this will
prevent performance regression of built-in formats. Is it
acceptable?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-09-08 21:08 ` Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-09-08 21:08 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Aug 13, 2025 at 11:37 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC+gf=3w7XiA4LQnvx0g@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 28 Jul 2025 22:19:36 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > The fields in 1 are mostly static fields, and the fields in 2 and 3
> > are likely to be accessed in hot functions during COPY FROM. Would it
> > be a good idea to restructure these fields so that we can hide the
> > fields in 1 from callback functions and having the fields in 3 in a
> > separate format-specific struct that can be accessed via an opaque
> > pointer? But could the latter change potentially cause performance
> > overheads?
>
> Yes. It changes memory layout (1 continuous memory chunk ->
> 2 separated memory chunks) and introduces indirect member
> accesses (x->y -> x->z->y).
I think fields accessed by the hot functions are limited. If we
assemble fields required by hot functions into one struct and pass it
to them can we deal with the latter? For example, we assemble the
fields in 3 I mentioned above (i.e., built-in format specific fields)
into say CopyFromStateBuiltin and pass it to CopyReadLine() function
and the following functions, instead of CopyFromState. Since there are
some places where we need to access to CopyFromState (e.g.,
CopyGetData()), CopyFromStateBuiltin needs to have a pointer to
CopyFromState as well.
> They may not have performance
> impact but we need to measure it if we want to use this
> approach.
Agreed.
> BTW, how about the following approach?
>
> copyapi.h:
>
> typedef struct CopyToStateData
> {
> /* public members */
> /* ... */
> } CopyToStateData;
>
> copyto.c:
>
> typedef struct CopyToStateInternalData
> {
> CopyToStateData parent;
>
> /* private members */
> /* ... */
> } CopyToStateInternalData;
>
> We export CopyToStateData only with public members. We don't
> export CopyToStateInternalData that has members only for
> built-in formats.
>
> CopyToStateInternalData has the same memory layout as
> CopyToStateData. So we can use CopyToStateInternalData as
> CopyToStateData.
>
> We use CopyToStateData not CopyToStateInternalData in public
> API. We cast CopyToStateData to CopyToStateInternalData when
> we need to use private members:
>
> static void
> CopySendData(CopyToState cstate, const void *databuf, int datasize)
> {
> CopyToStateInternal cstate_internal = (CopyToStateInternal) cstate;
> appendBinaryStringInfo(cstate_internal->fe_msgbuf, databuf, datasize);
> }
>
> It's still direct member access.
>
>
> With this approach, we can keep the same memory layout (1
> continuous memory chunk) and direct member access. I think
> that this approach doesn't have performance impact.
>
> See the attached patch for PoC of this approach.
>
> Drawback: This approach always allocates
> CopyToStateInternalData not CopyToStateData. So we need to
> allocate needless memories for extensions. But this will
> prevent performance regression of built-in formats. Is it
> acceptable?
While this approach could make sense to avoid potential performance
overheads for built-in format, I find it's somewhat odd that
extensions cannot allocate memory for its working state having
CopyToStateData as the base type.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-09-09 02:50 ` Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-09-09 02:50 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoCCjKA77xkUxx59qJ8an_G_58Mry_EtCEcFgd=g9N2xew@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 8 Sep 2025 14:08:16 -0700,
Masahiko Sawada <[email protected]> wrote:
>> > The fields in 1 are mostly static fields, and the fields in 2 and 3
>> > are likely to be accessed in hot functions during COPY FROM. Would it
>> > be a good idea to restructure these fields so that we can hide the
>> > fields in 1 from callback functions and having the fields in 3 in a
>> > separate format-specific struct that can be accessed via an opaque
>> > pointer? But could the latter change potentially cause performance
>> > overheads?
>>
>> Yes. It changes memory layout (1 continuous memory chunk ->
>> 2 separated memory chunks) and introduces indirect member
>> accesses (x->y -> x->z->y).
>
> I think fields accessed by the hot functions are limited. If we
> assemble fields required by hot functions into one struct and pass it
> to them can we deal with the latter? For example, we assemble the
> fields in 3 I mentioned above (i.e., built-in format specific fields)
> into say CopyFromStateBuiltin and pass it to CopyReadLine() function
> and the following functions, instead of CopyFromState. Since there are
> some places where we need to access to CopyFromState (e.g.,
> CopyGetData()), CopyFromStateBuiltin needs to have a pointer to
> CopyFromState as well.
It can change indirect member accesses (built-in format
specific members can be direct access but other members in
CopyFromState are indirect access) but it doesn't change 2
separated memory chunks.
If this approach has performance impact and it's caused by
indirect member accesses for built-in format specific
members, your suggestion will work. If performance impact is
caused by another reason, your suggestion may not work.
Anyway, we need to measure performance to proceed with this
approach. If we can confirm that this approach doesn't have
any performance impact, we can use the original your idea.
Do you have any idea how to measure performance of this
approach?
We did it when we introduce Copy{To,From}Routine. But it was
difficult to evaluate the results:
* I don't have machines for stable benchmark results
* We may not be able to use them for the final decision
* Most results showed performance improvement but
there was a result showed mysterious result[1]
[1] https://www.postgresql.org/message-id/flat/CAEG8a3LUBcvjwqgt6AijJmg67YN_b_NZ4Kzoxc_dH4rpAq0pKg%40mai...
>> Drawback: This approach always allocates
>> CopyToStateInternalData not CopyToStateData. So we need to
>> allocate needless memories for extensions. But this will
>> prevent performance regression of built-in formats. Is it
>> acceptable?
>
> While this approach could make sense to avoid potential performance
> overheads for built-in format, I find it's somewhat odd that
> extensions cannot allocate memory for its working state having
> CopyToStateData as the base type.
Is it important? We'll provide a opaque member for
extensions. Extensions should use the opaque member instead
of extending Copy{From,To}StateData.
I don't object your approach but we need a good way to
measure performance. If we use this approach, we can omit it
for now and we can revisit your approach later without
breaking compatibility. How about using this approach if we
can't find a good way to measure performance?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-09-09 20:15 ` Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-09-09 20:15 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Sep 8, 2025 at 7:50 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoCCjKA77xkUxx59qJ8an_G_58Mry_EtCEcFgd=g9N2xew@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 8 Sep 2025 14:08:16 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> > The fields in 1 are mostly static fields, and the fields in 2 and 3
> >> > are likely to be accessed in hot functions during COPY FROM. Would it
> >> > be a good idea to restructure these fields so that we can hide the
> >> > fields in 1 from callback functions and having the fields in 3 in a
> >> > separate format-specific struct that can be accessed via an opaque
> >> > pointer? But could the latter change potentially cause performance
> >> > overheads?
> >>
> >> Yes. It changes memory layout (1 continuous memory chunk ->
> >> 2 separated memory chunks) and introduces indirect member
> >> accesses (x->y -> x->z->y).
> >
> > I think fields accessed by the hot functions are limited. If we
> > assemble fields required by hot functions into one struct and pass it
> > to them can we deal with the latter? For example, we assemble the
> > fields in 3 I mentioned above (i.e., built-in format specific fields)
> > into say CopyFromStateBuiltin and pass it to CopyReadLine() function
> > and the following functions, instead of CopyFromState. Since there are
> > some places where we need to access to CopyFromState (e.g.,
> > CopyGetData()), CopyFromStateBuiltin needs to have a pointer to
> > CopyFromState as well.
>
> It can change indirect member accesses (built-in format
> specific members can be direct access but other members in
> CopyFromState are indirect access) but it doesn't change 2
> separated memory chunks.
Right. IIUC the latter point is related to cache locality. But I'm not
sure how much the latter point affects the performance as currently we
don't declare fields to CopyFromState while carefully considering the
cache locality even today. Separating a single memory into multiple
chunks could even have a positive effect on it.
>
> If this approach has performance impact and it's caused by
> indirect member accesses for built-in format specific
> members, your suggestion will work. If performance impact is
> caused by another reason, your suggestion may not work.
>
> Anyway, we need to measure performance to proceed with this
> approach. If we can confirm that this approach doesn't have
> any performance impact, we can use the original your idea.
>
> Do you have any idea how to measure performance of this
> approach?
I think we can start with measuring the entire COPY execution time
with several scenarios as we did previously. For example, reading a
huge file with a single column value and with many columns etc.
>
> We did it when we introduce Copy{To,From}Routine. But it was
> difficult to evaluate the results:
>
> * I don't have machines for stable benchmark results
> * We may not be able to use them for the final decision
> * Most results showed performance improvement but
> there was a result showed mysterious result[1]
Perhaps measuring cache-misses help to see how changes could affect
the performance?
>
> [1] https://www.postgresql.org/message-id/flat/CAEG8a3LUBcvjwqgt6AijJmg67YN_b_NZ4Kzoxc_dH4rpAq0pKg%40mai...
>
> >> Drawback: This approach always allocates
> >> CopyToStateInternalData not CopyToStateData. So we need to
> >> allocate needless memories for extensions. But this will
> >> prevent performance regression of built-in formats. Is it
> >> acceptable?
> >
> > While this approach could make sense to avoid potential performance
> > overheads for built-in format, I find it's somewhat odd that
> > extensions cannot allocate memory for its working state having
> > CopyToStateData as the base type.
>
> Is it important? We'll provide a opaque member for
> extensions. Extensions should use the opaque member instead
> of extending Copy{From,To}StateData.
I think yes, because it could be a blocker for future improvements
that might require a large field to CopyFrom/ToStateData.
> I don't object your approach but we need a good way to
> measure performance. If we use this approach, we can omit it
> for now and we can revisit your approach later without
> breaking compatibility. How about using this approach if we
> can't find a good way to measure performance?
I think it would be better to hear more opinions about this idea and
then make a decision, rather than basing our decision on whether or
not we can measure its performance, so we can be more confident in the
idea we have chosen. While this idea has the above downside, it could
make sense because we always allocate the entire CopyFrom/ToStateData
even today in spite of some fields being not used at all in binary
format and it requires less implementation costs to hide the
for-core-only fields. On the other hand, another possible idea is that
we have three different structs for categories 1 (core-only), 2 (core
and extensions), and 3 (extension-only), and expose only 2 that has a
void pointer to 3. The core can allocate the memory for 1 that embeds
2 at the beginning of the fields. While this design looks cleaner and
we can minimize overheads due to indirect references, it would require
more implementation costs. Which method we choose, I think we need
performance measurements in several scenarios to check if performance
regressions don't happen unexpectedly.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-09-10 02:40 ` Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-09-10 02:40 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoCidyfKcpf9-f2Np8kWgkM09c4TjnS1h1hcO_-CCbjeqw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 9 Sep 2025 13:15:43 -0700,
Masahiko Sawada <[email protected]> wrote:
>> I don't object your approach but we need a good way to
>> measure performance. If we use this approach, we can omit it
>> for now and we can revisit your approach later without
>> breaking compatibility. How about using this approach if we
>> can't find a good way to measure performance?
>
> I think it would be better to hear more opinions about this idea and
> then make a decision, rather than basing our decision on whether or
> not we can measure its performance, so we can be more confident in the
> idea we have chosen. While this idea has the above downside, it could
> make sense because we always allocate the entire CopyFrom/ToStateData
> even today in spite of some fields being not used at all in binary
> format and it requires less implementation costs to hide the
> for-core-only fields. On the other hand, another possible idea is that
> we have three different structs for categories 1 (core-only), 2 (core
> and extensions), and 3 (extension-only), and expose only 2 that has a
> void pointer to 3. The core can allocate the memory for 1 that embeds
> 2 at the beginning of the fields. While this design looks cleaner and
> we can minimize overheads due to indirect references, it would require
> more implementation costs. Which method we choose, I think we need
> performance measurements in several scenarios to check if performance
> regressions don't happen unexpectedly.
OK. So the next step is collecting more opinions, right?
Could you add key people in this area to Cc to hear their
opinions? I'm not familiar with key people in the PostgreSQL
community...
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-09-10 07:36 ` Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-09-10 07:36 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Sep 9, 2025 at 7:41 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoCidyfKcpf9-f2Np8kWgkM09c4TjnS1h1hcO_-CCbjeqw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 9 Sep 2025 13:15:43 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> I don't object your approach but we need a good way to
> >> measure performance. If we use this approach, we can omit it
> >> for now and we can revisit your approach later without
> >> breaking compatibility. How about using this approach if we
> >> can't find a good way to measure performance?
> >
> > I think it would be better to hear more opinions about this idea and
> > then make a decision, rather than basing our decision on whether or
> > not we can measure its performance, so we can be more confident in the
> > idea we have chosen. While this idea has the above downside, it could
> > make sense because we always allocate the entire CopyFrom/ToStateData
> > even today in spite of some fields being not used at all in binary
> > format and it requires less implementation costs to hide the
> > for-core-only fields. On the other hand, another possible idea is that
> > we have three different structs for categories 1 (core-only), 2 (core
> > and extensions), and 3 (extension-only), and expose only 2 that has a
> > void pointer to 3. The core can allocate the memory for 1 that embeds
> > 2 at the beginning of the fields. While this design looks cleaner and
> > we can minimize overheads due to indirect references, it would require
> > more implementation costs. Which method we choose, I think we need
> > performance measurements in several scenarios to check if performance
> > regressions don't happen unexpectedly.
>
> OK. So the next step is collecting more opinions, right?
>
> Could you add key people in this area to Cc to hear their
> opinions? I'm not familiar with key people in the PostgreSQL
> community...
How about another idea like we move format-specific data to another
struct that embeds CopyFrom/ToStateData at the first field and have
CopyFrom/ToStart callback return memory with the size of that
struct?It resolves the concerns about adding an extra indirection
layer and extensions doesn't need to allocate memory for unnecessary
fields (used only for built-in formats). While extensions can access
the internal fields I think we can live with that given that there are
some similar precedents such as table AM's scan descriptions.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-09-11 05:46 ` Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-09-11 05:46 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBb3t7EcsjYT4w68p9OfMNwWTYsbSVaSRY6DRhi7sNRFg@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 10 Sep 2025 00:36:38 -0700,
Masahiko Sawada <[email protected]> wrote:
> How about another idea like we move format-specific data to another
> struct that embeds CopyFrom/ToStateData at the first field and have
> CopyFrom/ToStart callback return memory with the size of that
> struct?It resolves the concerns about adding an extra indirection
> layer and extensions doesn't need to allocate memory for unnecessary
> fields (used only for built-in formats). While extensions can access
> the internal fields I think we can live with that given that there are
> some similar precedents such as table AM's scan descriptions.
The another idea looks like the following, right?
struct CopyToStateBuiltInData
{
struct CopyToStateData parent;
/* Members for built-in formats */
...;
}
typedef CopyToState *(*CopyToStart) (void);
CopyToState
BeginCopyTo(..., CopyToStart copy_to_start)
{
...;
/* Allocate workspace and zero all fields */
cstate = copy_to_start();
...;
}
This idea will almost work. But we can't know which
CopyToStart should be used before we parse "FORMAT" option
of COPY.
If we can iterate options twice in BeginCopy{To,From}(), we
can know it. For example:
BeginCopyTo(...)
{
...;
CopyToStart copy_to_start = NULL;
foreach(option, options)
{
DefElem *defel = lfirst_node(DefElem, option);
if (strcmp(defel->defname, "format") == 0)
{
char *fmt = defGetString(defel);
if (strcmp(fmt, "text") == 0 ||
strcmp(fmt, "csv") == 0 ||
strcmp(fmt, "binary") == 0) {
/* Use the builtin cstate */
} else {
copy_to_start = /* Detect CopyToStart for custom format */;
}
}
}
if (copy_to_start)
cstate = copy_to_start();
else
cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateBuiltInData));
...;
}
(It may be better that we add
Copy{To,From}Routine::Copy{To,From}Allocate() instead of
CopyToStart callback.)
I think that this is acceptable because this must be a light
process. This must not have negative performance impact.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-09-11 20:41 ` Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-09-11 20:41 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Sep 10, 2025 at 10:46 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBb3t7EcsjYT4w68p9OfMNwWTYsbSVaSRY6DRhi7sNRFg@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 10 Sep 2025 00:36:38 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > How about another idea like we move format-specific data to another
> > struct that embeds CopyFrom/ToStateData at the first field and have
> > CopyFrom/ToStart callback return memory with the size of that
> > struct?It resolves the concerns about adding an extra indirection
> > layer and extensions doesn't need to allocate memory for unnecessary
> > fields (used only for built-in formats). While extensions can access
> > the internal fields I think we can live with that given that there are
> > some similar precedents such as table AM's scan descriptions.
>
> The another idea looks like the following, right?
>
> struct CopyToStateBuiltInData
> {
> struct CopyToStateData parent;
>
> /* Members for built-in formats */
> ...;
> }
>
> typedef CopyToState *(*CopyToStart) (void);
>
> CopyToState
> BeginCopyTo(..., CopyToStart copy_to_start)
> {
> ...;
>
> /* Allocate workspace and zero all fields */
> cstate = copy_to_start();
> ...;
> }
Right.
> This idea will almost work. But we can't know which
> CopyToStart should be used before we parse "FORMAT" option
> of COPY.
>
> If we can iterate options twice in BeginCopy{To,From}(), we
> can know it. For example:
>
> BeginCopyTo(...)
> {
> ...;
>
> CopyToStart copy_to_start = NULL;
> foreach(option, options)
> {
> DefElem *defel = lfirst_node(DefElem, option);
>
> if (strcmp(defel->defname, "format") == 0)
> {
> char *fmt = defGetString(defel);
> if (strcmp(fmt, "text") == 0 ||
> strcmp(fmt, "csv") == 0 ||
> strcmp(fmt, "binary") == 0) {
> /* Use the builtin cstate */
> } else {
> copy_to_start = /* Detect CopyToStart for custom format */;
> }
> }
> }
> if (copy_to_start)
> cstate = copy_to_start();
> else
> cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateBuiltInData));
> ...;
> }
>
> (It may be better that we add
> Copy{To,From}Routine::Copy{To,From}Allocate() instead of
> CopyToStart callback.)
I think we can use a local variable of CopyFormatOptions and memcpy it
to the opts of the returned cstate.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-09-12 00:07 ` Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-09-12 00:07 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoCfqD=f2ELqPxg62+_QADhHi_kJXCDMhAerBtvxudd-xQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 11 Sep 2025 13:41:26 -0700,
Masahiko Sawada <[email protected]> wrote:
> I think we can use a local variable of CopyFormatOptions and memcpy it
> to the opts of the returned cstate.
It'll work too. Can we proceed this proposal with this
approach? Should we collect more opinions before we proceed?
If so, Could you add key people in this area to Cc to hear
their opinions?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-09-15 17:00 ` Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-09-15 17:00 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Sep 11, 2025 at 5:07 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoCfqD=f2ELqPxg62+_QADhHi_kJXCDMhAerBtvxudd-xQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 11 Sep 2025 13:41:26 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > I think we can use a local variable of CopyFormatOptions and memcpy it
> > to the opts of the returned cstate.
>
> It'll work too. Can we proceed this proposal with this
> approach? Should we collect more opinions before we proceed?
> If so, Could you add key people in this area to Cc to hear
> their opinions?
Since we don't have a single decision-maker, we should proceed through
consensus-building and careful evaluation of each approach. I see that
several senior hackers are already included in this thread, which is
excellent. Since you and I, who have been involved in these
discussions, agreed with this approach, I believe we can proceed with
this direction. If anyone proposes alternative solutions that we find
more compelling, we might have to change the approach.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-10-03 07:06 ` Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-10-03 07:06 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoADXWgdizS0mV5w8wdfftDRsm8sUtNW=CzYYS1OhjFD2A@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Sep 2025 10:00:18 -0700,
Masahiko Sawada <[email protected]> wrote:
>> > I think we can use a local variable of CopyFormatOptions and memcpy it
>> > to the opts of the returned cstate.
>>
>> It'll work too. Can we proceed this proposal with this
>> approach? Should we collect more opinions before we proceed?
>> If so, Could you add key people in this area to Cc to hear
>> their opinions?
>
> Since we don't have a single decision-maker, we should proceed through
> consensus-building and careful evaluation of each approach. I see that
> several senior hackers are already included in this thread, which is
> excellent. Since you and I, who have been involved in these
> discussions, agreed with this approach, I believe we can proceed with
> this direction. If anyone proposes alternative solutions that we find
> more compelling, we might have to change the approach.
OK. There is no objection for now.
How about the attached patch? The patch uses the approach
only for CopyToStateData. If this looks good, I can do it
for CopyFromStateData too.
This patch splits CopyToStateData to
* CopyToStateData
* CopyToStateInternalData
* CopyToStateBuiltinData
structs.
This is based on the category described in
https://www.postgresql.org/message-id/flat/CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC%2Bgf%3D3w7XiA4LQnvx0g%4...
:
> 1. fields used only the core (not by format callback)
> 2. fields used by both the core and format callbacks
> 3. built-in format specific fields (mostly for text and csv)
CopyToStateInternalData is for 1.
CopyToStateData is for 2.
CopyToStateBuiltinData is for 3.
This patch adds CopyToRoutine::CopyToGetStateSize() that
returns size of state struct for the routine. For example,
Built-in formats use sizeof(CopyToStateBuiltinData) for it.
BeginCopyTo() allocates sizeof(CopyToStateInternalData) +
CopyToGetStateSize() size continuous memory and uses the
front part as CopyToStateInternalData and the back part as
CopyToStateData/CopyToStateBuilinData.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-10-13 21:40 ` Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-10-13 21:40 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; Andres Freund <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Oct 3, 2025 at 12:06 AM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoADXWgdizS0mV5w8wdfftDRsm8sUtNW=CzYYS1OhjFD2A@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Sep 2025 10:00:18 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> >> > I think we can use a local variable of CopyFormatOptions and memcpy it
> >> > to the opts of the returned cstate.
> >>
> >> It'll work too. Can we proceed this proposal with this
> >> approach? Should we collect more opinions before we proceed?
> >> If so, Could you add key people in this area to Cc to hear
> >> their opinions?
> >
> > Since we don't have a single decision-maker, we should proceed through
> > consensus-building and careful evaluation of each approach. I see that
> > several senior hackers are already included in this thread, which is
> > excellent. Since you and I, who have been involved in these
> > discussions, agreed with this approach, I believe we can proceed with
> > this direction. If anyone proposes alternative solutions that we find
> > more compelling, we might have to change the approach.
>
> OK. There is no objection for now.
>
> How about the attached patch? The patch uses the approach
> only for CopyToStateData. If this looks good, I can do it
> for CopyFromStateData too.
>
> This patch splits CopyToStateData to
>
> * CopyToStateData
> * CopyToStateInternalData
> * CopyToStateBuiltinData
>
> structs.
>
> This is based on the category described in
> https://www.postgresql.org/message-id/flat/CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC%2Bgf%3D3w7XiA4LQnvx0g%4...
> :
>
> > 1. fields used only the core (not by format callback)
> > 2. fields used by both the core and format callbacks
> > 3. built-in format specific fields (mostly for text and csv)
>
> CopyToStateInternalData is for 1.
> CopyToStateData is for 2.
> CopyToStateBuiltinData is for 3.
>
>
> This patch adds CopyToRoutine::CopyToGetStateSize() that
> returns size of state struct for the routine. For example,
> Built-in formats use sizeof(CopyToStateBuiltinData) for it.
>
> BeginCopyTo() allocates sizeof(CopyToStateInternalData) +
> CopyToGetStateSize() size continuous memory and uses the
> front part as CopyToStateInternalData and the back part as
> CopyToStateData/CopyToStateBuilinData.
Thank you for drafting the idea!
The patch refactors the CopyToStateData so that we can both hide
internal-use-only fields from extensions and extension can use its own
state data, while not adding extra indirection layers. TBH I'm really
not sure we must fully hide internal fields from extensions. Other
extendable components seem not to strictly hide internal information
from extensions. I'd suggest starting with only the latter point. That
is, we merge fields in CopyToStateInternalData to CopyToStateData.
What do you think?
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-10-14 02:15 ` Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-10-14 02:15 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700,
Masahiko Sawada <[email protected]> wrote:
> The patch refactors the CopyToStateData so that we can both hide
> internal-use-only fields from extensions and extension can use its own
> state data, while not adding extra indirection layers. TBH I'm really
> not sure we must fully hide internal fields from extensions. Other
> extendable components seem not to strictly hide internal information
> from extensions. I'd suggest starting with only the latter point. That
> is, we merge fields in CopyToStateInternalData to CopyToStateData.
> What do you think?
OK. Let's follow the existing style. How about the attached
patch? It merges CopyToStateInternalData to CopyToStateData.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-10-29 20:41 ` Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-10-29 20:41 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Oct 13, 2025 at 7:15 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > The patch refactors the CopyToStateData so that we can both hide
> > internal-use-only fields from extensions and extension can use its own
> > state data, while not adding extra indirection layers. TBH I'm really
> > not sure we must fully hide internal fields from extensions. Other
> > extendable components seem not to strictly hide internal information
> > from extensions. I'd suggest starting with only the latter point. That
> > is, we merge fields in CopyToStateInternalData to CopyToStateData.
> > What do you think?
>
> OK. Let's follow the existing style. How about the attached
> patch? It merges CopyToStateInternalData to CopyToStateData.
>
The basic idea of this patch makes sense to me.
Andres, I believe that the current idea deals with your concerns about
performance overheads. Particularly, we separate format-specific
fields (c.f. CopyToStateBuiltinData struct in the patch) from the
commonly-used fields (c.f., CopyToStateData struct), but the whole
fields are stored in the contiguous memory. While the patch needs to
be polished much, could you review if the basic idea of this patch
addresses your concern?
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-11-14 20:19 ` Masahiko Sawada <[email protected]>
2025-11-17 06:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
0 siblings, 2 replies; 125+ messages in thread
From: Masahiko Sawada @ 2025-11-14 20:19 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Oct 29, 2025 at 1:41 PM Masahiko Sawada <[email protected]> wrote:
>
> On Mon, Oct 13, 2025 at 7:15 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700,
> > Masahiko Sawada <[email protected]> wrote:
> >
> > > The patch refactors the CopyToStateData so that we can both hide
> > > internal-use-only fields from extensions and extension can use its own
> > > state data, while not adding extra indirection layers. TBH I'm really
> > > not sure we must fully hide internal fields from extensions. Other
> > > extendable components seem not to strictly hide internal information
> > > from extensions. I'd suggest starting with only the latter point. That
> > > is, we merge fields in CopyToStateInternalData to CopyToStateData.
> > > What do you think?
> >
> > OK. Let's follow the existing style. How about the attached
> > patch? It merges CopyToStateInternalData to CopyToStateData.
> >
>
> The basic idea of this patch makes sense to me.
>
This thread has involved extensive discussion, and the patch needs to
be rebased. I'd like to summarize the current status of this patch and
our discussions. I've attached updated patches that implement the
whole ideas of this feature to help provide a clearer overall picture.
In commits 2e4127b6d and 7717f6300, we refactored COPY TO/FROM code to
use a set of callbacks for format-specific operations like data
parsing. These callbacks are currently not exposed and are only used
by built-in formats. The next step is to allow extensions to register
their own format implementations, which these attached patches
accomplish. The registration API can be called in _PG_init() as
follows to enable users to specify a custom format name ('jsonlines'
in this example) in the FORMAT option:
RegisterCopyCustomFormat("jsonlines",
&JsonLinesCopyFromRoutine,
&JsonLinesCopyToRoutine);
However, before introducing the registration API, we need to resolve
an issue with how we currently use a monolithic struct
(Copy{From,To}StateData) to store COPY TO/FROM state data. This struct
currently contains both format-agnostic fields (e.g., target relation
and source file) and format-specific fields (e.g., input buffers and
EOL type). Patches 0001 and 0002 reorganize these fields to separate
them. Specifically, format-specific fields are moved to a new struct
while Copy{From,To}StateData retains format-agnostic fields, as shown
here (e.g., COPY TO case):
typedef struct CopyToStateData
{
/* format-specific routines */
const struct CopyToRoutine *routine;
/* low-level state data */
CopyDest copy_dest; /* type of copy source/destination */
FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */
StringInfo fe_msgbuf; /* used for all dests during COPY TO */
(snip)
/*
* Working state
*/
MemoryContext copycontext; /* per-copy execution context */
FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
} CopyToStateData;
typedef struct CopyToStateTextLike
{
CopyToStateData base; /* embedded */
int file_encoding;
bool need_transcoding;
bool encoding_embeds_ascii;
} CopyToStateTextLike;
Extensions must specify their required state struct size (like
CopyToStateTextLike for built-in formats) using new callbacks
Copy{From,To}EstimateStateSpace, allowing the core to allocate the
appropriate amount of memory. This approach offers two advantages:
- Format processing implementations only use the memory they need
- No additional pointer traversal compared to using an opaque pointer
for format-specific data
Patches 0003 through 0006 implement the following:
0003: Introduces the RegisterCustomCopyFormat() API
0004 and 0005: Enable custom format implementations to register their
own COPY command options
0006: Adds regression tests
With these patches, here's what we can do using the 'jsonlines'
format extension:
-- COPY TO
CREATE TABLE jl (id int, a text, b jsonb);
INSERT INTO jl VALUES (1, 'hello', '{"test" : [1, true, {"num" :
42}]}'::jsonb), (2, 'hello world', 'true'), (999, null, '{"a" : 1}');
TABLE jl;
id | a | b
-----+-------------+----------------------------------
1 | hello | {"test": [1, true, {"num": 42}]}
2 | hello world | true
999 | | {"a": 1}
(3 rows)
COPY jl TO '/tmp/test.jsonl' WITH (format 'jsonlines');
\! cat /tmp/test.jsonl
{"id":1,"a":"hello","b":{"test": [1, true, {"num": 42}]}}
{"id":2,"a":"hello world","b":true}
{"id":999,"a":null,"b":{"a": 1}}
-- COPY FROM
CREATE TABLE jl_load (id int, a text, b jsonb);
COPY jl_load FROM '/tmp/test.jsonl' WITH (format 'jsonlines');
-- COPY TO/FROM with custom options
COPY jl TO '/tmp/jl.jsonl.gz' WITH (format 'jsonlines', compression
'gzip', compression_detail 'level=2');
COPY jl FROM '/tmp/jl.jsonl.gz' WITH (format 'jsonlines');
To demonstrate the functionality of both current and new APIs,
Suto-san and I have created several experimental custom COPY format
extensions:
Apache Arrow (developed by Sutou-san): https://github.com/kou/pg-copy-arrow
JSON Lines (developed by me):
https://github.com/MasahikoSawada/pg_custom_copy_formats/blob/master/jsonlines.c
These implementations serve as good examples of how extensions can use
these APIs to define custom COPY formats.
After offline discussions with Sutou-san, we believe the current APIs
work well, particularly for text-based formats, though we still need
to verify there are no performance regressions.
One potential improvement would be adding support for random file
access in COPY FROM operations. For example, with parquet files, it
would be much more efficient to read the footer section first since it
contains metadata, allowing selective reading of necessary file
sections. The current sequential read API (CopyFromGetData()) requires
reading all data to access the metadata.
For future consideration, we could look into supporting file
reading/writing from external sources like S3. While this is outside
the scope of this patch, we discussed that allowing the core to
delegate I/O operations to custom format implementations might be a
good starting point. We can discuss this in a separate thread.
I welcome your feedback on these proposed changes and APIs to help
move this patch forward.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] 0003-Support-custom-COPY-format-for-COPY-FROM-and-COPY-TO.patch (9.6K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/2-0003-Support-custom-COPY-format-for-COPY-FROM-and-COPY-TO.patch)
download | inline diff:
From c1ab564adbe8db74c97c1d0e4ee6ec552457474c Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 10 Nov 2025 11:26:10 -0800
Subject: [PATCH 3/6] Support custom COPY format for COPY FROM and COPY TO.
This commit enables extensions to register their own COPY format
implementation for COPY TO and COPY FROM. If extensions register the
custom format during _PG_init_ time, the format would be available for
all backends whereas the backend can LOAD the custom format.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
src/backend/commands/Makefile | 1 +
src/backend/commands/copy_custom_format.c | 142 ++++++++++++++++++++++
src/backend/commands/copyfrom.c | 7 ++
src/backend/commands/copyfromparse.c | 9 ++
src/backend/commands/copyto.c | 17 +++
src/backend/commands/meson.build | 1 +
src/include/commands/copy.h | 6 +
src/include/commands/copyapi.h | 9 ++
8 files changed, 192 insertions(+)
create mode 100644 src/backend/commands/copy_custom_format.c
diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile
index f99acfd2b4b..74668d4ac9d 100644
--- a/src/backend/commands/Makefile
+++ b/src/backend/commands/Makefile
@@ -27,6 +27,7 @@ OBJS = \
copyfrom.o \
copyfromparse.o \
copyto.o \
+ copy_custom_format.o \
createas.o \
dbcommands.o \
define.o \
diff --git a/src/backend/commands/copy_custom_format.c b/src/backend/commands/copy_custom_format.c
new file mode 100644
index 00000000000..8bef6e779ac
--- /dev/null
+++ b/src/backend/commands/copy_custom_format.c
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * copy_custom_format.c
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/commands/copy_custom_format.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/copyapi.h"
+#include "utils/memutils.h"
+
+
+typedef struct CopyCustomFormat
+{
+ const char *fmt_name;
+ const CopyFromRoutine *from_routine;
+ const CopyToRoutine *to_routine;
+} CopyCustomFormat;
+
+static CopyCustomFormat * CopyCustomFormatArray = NULL;
+static int CopyCustomFormatAssigned = 0;
+static int CopyCustomFormatAllocated = 0;
+
+static char *BuiltinFormatNames[] = {
+ "text", "csv", "binary"
+};
+
+void
+RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine,
+ const CopyToRoutine *to_routine)
+{
+ CopyCustomFormat *entry;
+
+ /* Check the duplication with built-in formats */
+ for (int i = 0; i < lengthof(BuiltinFormatNames); i++)
+ {
+ if (strcmp(BuiltinFormatNames[i], fmt_name) == 0)
+ ereport(ERROR,
+ errmsg("failed to register custom COPY format \"%s\"", fmt_name),
+ errdetail("COPY format \"%s\" is a builtin format",
+ BuiltinFormatNames[i]));
+
+ }
+
+ /* Check the duplication with the registered custom formats */
+ if (FindCustomCopyFormat(fmt_name))
+ ereport(ERROR,
+ errmsg("failed to register custom COPY format \"%s\"", fmt_name),
+ errdetail("Custom COPY format \"%s\" already registered",
+ fmt_name));
+
+ /* If there is no array yet, create one */
+ if (CopyCustomFormatArray == NULL)
+ {
+ CopyCustomFormatAllocated = 16;
+ CopyCustomFormatArray = (CopyCustomFormat *)
+ MemoryContextAlloc(TopMemoryContext,
+ CopyCustomFormatAllocated * sizeof(CopyCustomFormat));
+ }
+
+ /* If there's an array but it's current full, expand it */
+ if (CopyCustomFormatAssigned >= CopyCustomFormatAllocated)
+ {
+ int i = pg_nextpower2_32(CopyCustomFormatAllocated + 1);
+
+ CopyCustomFormatArray = (CopyCustomFormat *)
+ repalloc(CopyCustomFormatArray, i * sizeof(CopyCustomFormat));
+ CopyCustomFormatAllocated = i;
+ }
+
+ Assert(to_routine != NULL || from_routine != NULL);
+ Assert((to_routine == NULL) ||
+ (to_routine->CopyToEstimateStateSpace != NULL &&
+ to_routine->CopyToOutFunc != NULL &&
+ to_routine->CopyToStart != NULL &&
+ to_routine->CopyToOneRow != NULL &&
+ to_routine->CopyToEnd != NULL));
+ Assert((from_routine == NULL) ||
+ (from_routine->CopyFromEstimateStateSpace != NULL &&
+ from_routine->CopyFromInFunc != NULL &&
+ from_routine->CopyFromStart != NULL &&
+ from_routine->CopyFromOneRow != NULL &&
+ from_routine->CopyFromEnd != NULL));
+
+ entry = &CopyCustomFormatArray[CopyCustomFormatAssigned++];
+ entry->fmt_name = fmt_name;
+ entry->from_routine = from_routine;
+ entry->to_routine = to_routine;
+}
+
+/*
+ * Returns true if the given custom format name is registered.
+ */
+bool
+FindCustomCopyFormat(const char *fmt_name)
+{
+ for (int i = 0; i < CopyCustomFormatAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0)
+ return true;
+ }
+
+ return false;
+}
+
+bool
+GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine)
+{
+ for (int i = 0; i < CopyCustomFormatAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0)
+ {
+ *to_routine = CopyCustomFormatArray[i].to_routine;
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool
+GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine)
+{
+ for (int i = 0; i < CopyCustomFormatAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0)
+ {
+ *from_routine = CopyCustomFormatArray[i].from_routine;
+ return true;
+ }
+ }
+
+ return false;
+}
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 0c51e5ba5e1..592be4fcb5d 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1619,6 +1619,13 @@ create_copyfrom_state(ParseState *pstate, List *options)
routine = &CopyFromRoutineCSV;
else if (strcmp(fmt, "binary") == 0)
routine = &CopyFromRoutineBinary;
+ else if (GetCustomCopyFromRoutine(fmt, &routine))
+ {
+ if (routine == NULL)
+ ereport(ERROR,
+ errmsg("COPY format \"%s\" does not support COPY FROM", fmt),
+ parser_errposition(pstate, defel->location));
+ }
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 669dbfb8459..30d8047f21a 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -228,6 +228,15 @@ ReceiveCopyBinaryHeader(CopyFromState ccstate)
}
}
+/*
+ * Exporting CopyGetData() function for custom COPY FROM format implementations.
+ */
+int
+CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
+{
+ return CopyGetData(cstate, databuf, minread, maxread);
+}
+
/*
* CopyGetData reads data from the source (file or frontend)
*
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6a0a66507ba..b1b3ae141eb 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -435,6 +435,16 @@ CopySendChar(CopyToState cstate, char c)
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
}
+/*
+ * Exporting CopySendEndOfRow() function for custom COPY TO format
+ * implementations.
+ */
+void
+CopyToFlushData(CopyToState cstate)
+{
+ CopySendEndOfRow(cstate);
+}
+
static void
CopySendEndOfRow(CopyToState cstate)
{
@@ -631,6 +641,13 @@ create_copyto_state(ParseState *pstate, List *options)
routine = &CopyToRoutineCSV;
else if (strcmp(fmt, "binary") == 0)
routine = &CopyToRoutineBinary;
+ else if (GetCustomCopyToRoutine(fmt, &routine))
+ {
+ if (routine == NULL)
+ ereport(ERROR,
+ errmsg("COPY format \"%s\" does not support COPY TO", fmt),
+ parser_errposition(pstate, defel->location));
+ }
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build
index 9f640ad4810..827f809eb53 100644
--- a/src/backend/commands/meson.build
+++ b/src/backend/commands/meson.build
@@ -15,6 +15,7 @@ backend_sources += files(
'copyfrom.c',
'copyfromparse.c',
'copyto.c',
+ 'copy_custom_format.c',
'createas.c',
'dbcommands.c',
'define.c',
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 30a1d2bff6e..82f07b05823 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -120,6 +120,12 @@ extern uint64 CopyFrom(CopyFromState cstate);
extern DestReceiver *CreateCopyDestReceiver(void);
+extern void ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out,
+ bool is_from, List **other_options, ParseState *pstate);
+
+/* defined in copy_custom_format.c */
+extern bool FindCustomCopyFormat(const char *fmt_name);
+
/*
* internal prototypes
*/
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index c3d2199a0b6..2090a4e1c61 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -111,4 +111,13 @@ typedef struct CopyFromRoutine
*/
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
+
+extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread);
+extern void CopyToFlushData(CopyToState cstate);
+
+extern void RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine,
+ const CopyToRoutine *to_routine);
+extern bool GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine);
+extern bool GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine);
+
#endif /* COPYAPI_H */
--
2.47.3
[application/octet-stream] 0006-Add-regression-test-module-for-custom-COPY-format.patch (17.0K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/3-0006-Add-regression-test-module-for-custom-COPY-format.patch)
download | inline diff:
From 2718f015e44354dca5ba99927d6ef0aa2dbb3b1f Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 10 Nov 2025 22:18:28 -0800
Subject: [PATCH 6/6] Add regression test module for custom COPY format.
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../test_copy_custom_format/.gitignore | 3 +
.../modules/test_copy_custom_format/Makefile | 21 ++
.../expected/test_copy_custom_format.out | 97 ++++++++
.../test_copy_custom_format/meson.build | 35 +++
.../sql/test_copy_custom_format.sql | 34 +++
.../test_copy_custom_format.c | 217 ++++++++++++++++++
.../test_copy_custom_format.conf | 1 +
9 files changed, 410 insertions(+)
create mode 100644 src/test/modules/test_copy_custom_format/.gitignore
create mode 100644 src/test/modules/test_copy_custom_format/Makefile
create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
create mode 100644 src/test/modules/test_copy_custom_format/meson.build
create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c
create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.conf
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 902a7954101..710ccb74990 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -19,6 +19,7 @@ SUBDIRS = \
test_bitmapset \
test_bloomfilter \
test_copy_callbacks \
+ test_copy_custom_format \
test_custom_rmgrs \
test_ddl_deparse \
test_dsa \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 14fc761c4cf..5a2e4e03b80 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -18,6 +18,7 @@ subdir('test_binaryheap')
subdir('test_bitmapset')
subdir('test_bloomfilter')
subdir('test_copy_callbacks')
+subdir('test_copy_custom_format')
subdir('test_custom_rmgrs')
subdir('test_ddl_deparse')
subdir('test_dsa')
diff --git a/src/test/modules/test_copy_custom_format/.gitignore b/src/test/modules/test_copy_custom_format/.gitignore
new file mode 100644
index 00000000000..8d3af519eb1
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/.gitignore
@@ -0,0 +1,3 @@
+/log/
+/results/
+/tmp_check/
\ No newline at end of file
diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile
new file mode 100644
index 00000000000..8ad1cebaba6
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/Makefile
@@ -0,0 +1,21 @@
+# src/test/modules/test_copy_custom_format/Makefile
+
+MODULE_big = test_copy_custom_format
+OBJS = \
+ $(WIN32RES) \
+ test_copy_custom_format.o
+PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT"
+
+REGRESS = test_copy_custom_format
+REGRESS_OPTS = --temp-config $(top_srcdir)/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_copy_custom_format
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
new file mode 100644
index 00000000000..a91ac293447
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
@@ -0,0 +1,97 @@
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY copy_data FROM stdin WITH (FORMAT 'test_format');
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int 0 common_bool 0 from_str "(null)"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data TO stdout WITH (FORMAT 'test_format');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes: 3
+NOTICE: common_int 0 common_bool 0 to_str "(null)"
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+-- Error
+COPY copy_data FROM stdin WITH (FORMAT 'error');
+ERROR: COPY format "error" not recognized
+LINE 1: COPY copy_data FROM stdin WITH (FORMAT 'error');
+ ^
+COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error');
+ERROR: conflicting or redundant options
+LINE 1: COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error...
+ ^
+-- Option handling, COPY FROM
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 10);
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int 10 common_bool 0 from_str "(null)"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int -10);
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int -10 common_bool 0 from_str "(null)"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 'a'); -- error
+ERROR: common_int requires an integer value
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'true');
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int 0 common_bool 1 from_str "(null)"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'false');
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int 0 common_bool 0 from_str "(null)"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'hello'); -- error
+ERROR: common_bool requires a Boolean value
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 100, common_bool false, from_str 'from option');
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes: 3
+NOTICE: common_int 100 common_bool 0 from_str "from option"
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'option'); -- error
+ERROR: COPY format "invalid" not recognized
+LINE 1: ... copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'o...
+ ^
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', format 'text'); -- error
+ERROR: conflicting or redundant options
+LINE 1: ... copy_data FROM stdin WITH (FORMAT 'test_format', format 'te...
+ ^
+-- Option handling, COPY FROM
+COPY copy_data TO stdout WITH (FORMAT 'test_format', common_int -10, common_bool false, to_str 'to option');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes: 3
+NOTICE: common_int -10 common_bool 0 to_str "to option"
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (FORMAT 'test_format', from_str 'to option'); -- error
+ERROR: COPY format "from_str" not recognized
+LINE 1: ...Y copy_data TO stdout WITH (FORMAT 'test_format', from_str '...
+ ^
diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build
new file mode 100644
index 00000000000..6cc6c8b60fd
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/meson.build
@@ -0,0 +1,35 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+test_copy_custom_format_sources = files(
+'test_copy_custom_format.c',
+)
+
+if host_system == 'windows'
+ test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_copy_custom_format',
+ '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',])
+endif
+
+test_copy_custom_format = shared_module('test_copy_custom_format',
+ test_copy_custom_format_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_copy_custom_format
+
+tests += {
+ 'name': 'test_copy_custom_format',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_copy_custom_format',
+ ],
+ 'regress_args': [
+ '--temp-config', files('test_copy_custom_format.conf'),
+ ],
+ # Disabled because these tests require
+ # "shared_preload_libraries=test_custom_copy_format", which typical
+ # runningcheck users do not have (e.g. buildfarm clients).
+ 'runningcheck': false,
+ },
+}
diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
new file mode 100644
index 00000000000..b6564f0355b
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
@@ -0,0 +1,34 @@
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+
+COPY copy_data FROM stdin WITH (FORMAT 'test_format');
+\.
+COPY copy_data TO stdout WITH (FORMAT 'test_format');
+
+-- Error
+COPY copy_data FROM stdin WITH (FORMAT 'error');
+COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error');
+
+
+-- Option handling, COPY FROM
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 10);
+\.
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int -10);
+\.
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 'a'); -- error
+
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'true');
+\.
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'false');
+\.
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'hello'); -- error
+
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 100, common_bool false, from_str 'from option');
+\.
+
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'option'); -- error
+COPY copy_data FROM stdin WITH (FORMAT 'test_format', format 'text'); -- error
+
+-- Option handling, COPY FROM
+COPY copy_data TO stdout WITH (FORMAT 'test_format', common_int -10, common_bool false, to_str 'to option');
+COPY copy_data TO stdout WITH (FORMAT 'test_format', from_str 'to option'); -- error
diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
new file mode 100644
index 00000000000..a63390e875b
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
@@ -0,0 +1,217 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_copy_custom_format.c
+ * Code for testing custom COPY format.
+ *
+ * Portions Copyright (c) 2025, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_copy_custom_format/test_copy_custom_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copystate.h"
+#include "commands/copyapi.h"
+#include "commands/defrem.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct TestCopyCommonOption
+{
+ int common_int;
+ bool common_bool;
+} TestCopyCommonOption;
+
+typedef struct TestCopyFromState
+{
+ CopyFromStateData base;
+
+ TestCopyCommonOption common_opts;
+ char *from_str_option;
+} TestCopyFromState;
+
+typedef struct TestCopyToState
+{
+ CopyToStateData base;
+
+ TestCopyCommonOption common_opts;
+ char *to_str_option;
+} TestCopyToState;
+
+static Size
+TestCopyToEsimateSpace(void)
+{
+ return sizeof(TestCopyToState);
+}
+
+static Size
+TestCopyFromEsimateSpace(void)
+{
+ return sizeof(TestCopyFromState);
+}
+
+static bool
+TestCopyProcessCommonOption(TestCopyCommonOption * opt, DefElem *option)
+{
+ if (strcmp(option->defname, "common_int") == 0)
+ {
+ int val = defGetInt32(option);
+
+ opt->common_int = val;
+
+ return true;
+ }
+ else if (strcmp(option->defname, "common_bool") == 0)
+ {
+ bool val = defGetBoolean(option);
+
+ opt->common_bool = val;
+
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+TestCopyFromProcessOneOption(CopyFromState ccstate, DefElem *option)
+{
+ TestCopyFromState *cstate = (TestCopyFromState *) ccstate;
+
+ if (TestCopyProcessCommonOption(&cstate->common_opts, option))
+ {
+ return true;
+ }
+ else if (strcmp(option->defname, "from_str") == 0)
+ {
+ char *val = defGetString(option);
+
+ cstate->from_str_option = val;
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+TestCopyToProcessOneOption(CopyToState ccstate, DefElem *option)
+{
+ TestCopyToState *cstate = (TestCopyToState *) ccstate;
+
+ if (TestCopyProcessCommonOption(&cstate->common_opts, option))
+ {
+ return true;
+ }
+ else if (strcmp(option->defname, "to_str") == 0)
+ {
+ char *val = defGetString(option);
+
+ cstate->to_str_option = val;
+ return true;
+ }
+
+ return false;
+}
+
+static void
+TestCopyFromInFunc(CopyFromState cstate, Oid atttypid,
+ FmgrInfo *finfo, Oid *typioparam)
+{
+ ereport(NOTICE,
+ errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+TestCopyFromStart(CopyFromState ccstate, TupleDesc tupDesc)
+{
+ TestCopyFromState *cstate = (TestCopyFromState *) ccstate;
+
+ ereport(NOTICE,
+ errmsg("CopyFromStart: the number of attributes: %d", tupDesc->natts));
+ ereport(NOTICE,
+ errmsg("common_int %d common_bool %d from_str \"%s\"",
+ cstate->common_opts.common_int,
+ cstate->common_opts.common_bool,
+ cstate->from_str_option));
+}
+
+static bool
+TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls,
+ CopyFromRowInfo * rowinfo)
+{
+ ereport(NOTICE,
+ errmsg("CopyFromOneRow"));
+
+ return false;
+}
+
+static void
+TestCopyFromEnd(CopyFromState cstate)
+{
+ ereport(NOTICE,
+ errmsg("CopyFromEnd"));
+}
+
+static void
+TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo)
+{
+ ereport(NOTICE,
+ errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+TestCopyToStart(CopyToState ccstate, TupleDesc tupDesc)
+{
+ TestCopyToState *cstate = (TestCopyToState *) ccstate;
+
+ ereport(NOTICE,
+ errmsg("CopyToStart: the number of attributes: %d", tupDesc->natts));
+ ereport(NOTICE,
+ errmsg("common_int %d common_bool %d to_str \"%s\"",
+ cstate->common_opts.common_int,
+ cstate->common_opts.common_bool,
+ cstate->to_str_option));
+}
+
+static void
+TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid)));
+}
+
+static void
+TestCopyToEnd(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToEnd")));
+}
+
+static const CopyToRoutine CopyToRoutineTestCopyFormat = {
+ .CopyToEstimateStateSpace = TestCopyToEsimateSpace,
+ .CopyToProcessOneOption = TestCopyToProcessOneOption,
+ .CopyToOutFunc = TestCopyToOutFunc,
+ .CopyToStart = TestCopyToStart,
+ .CopyToOneRow = TestCopyToOneRow,
+ .CopyToEnd = TestCopyToEnd,
+};
+
+
+static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
+ .CopyFromEstimateStateSpace = TestCopyFromEsimateSpace,
+ .CopyFromProcessOneOption = TestCopyFromProcessOneOption,
+ .CopyFromInFunc = TestCopyFromInFunc,
+ .CopyFromStart = TestCopyFromStart,
+ .CopyFromOneRow = TestCopyFromOneRow,
+ .CopyFromEnd = TestCopyFromEnd,
+};
+
+void
+_PG_init(void)
+{
+ RegisterCopyCustomFormat("test_format",
+ &CopyFromRoutineTestCopyFormat,
+ &CopyToRoutineTestCopyFormat);
+}
diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf b/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf
new file mode 100644
index 00000000000..fb7cf22a3b1
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf
@@ -0,0 +1 @@
+shared_preload_libraries = 'test_copy_custom_format'
--
2.47.3
[application/octet-stream] 0004-Refactor-Copy-option-processing.patch (6.9K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/4-0004-Refactor-Copy-option-processing.patch)
download | inline diff:
From 362ceb242c7b026bcbb077c910b26512b08e1e9d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 10 Nov 2025 14:53:51 -0800
Subject: [PATCH 4/6] Refactor Copy option processing.
This commits refactors the COPY option handling to support custom COPY
option callback. Since extensions can define custom COPY format with
its own state data, it's necessary to pass the state data to the
option process callback. This commits separates ProcessCopyOptions
into ProcessCopyFromOptions and ProcessCopyToOptions and passes a
pointer to Copy{From,To}State respectively.
The subsequent patch introduces a new callback to allow extensions to
define their own option processing callback.
---
contrib/file_fdw/file_fdw.c | 2 +-
src/backend/commands/copy.c | 18 ++++---------
src/backend/commands/copyfrom.c | 31 ++++++++++++++++++++++-
src/backend/commands/copyto.c | 32 +++++++++++++++++++++++-
src/include/commands/copyfrom_internal.h | 2 ++
5 files changed, 69 insertions(+), 16 deletions(-)
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index af2d86e5b5c..447fcb44241 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -338,7 +338,7 @@ file_fdw_validator(PG_FUNCTION_ARGS)
/*
* Now apply the core COPY code's validation logic for more checks.
*/
- ProcessCopyOptions(NULL, NULL, true, other_options);
+ ProcessCopyFromOptions(NULL, other_options, NULL);
/*
* Either filename or program option is required for file_fdw foreign
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 28e878c3688..d4ee649c3c6 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -546,10 +546,8 @@ defGetCopyLogVerbosityChoice(DefElem *def, ParseState *pstate)
* self-consistency of the options list.
*/
void
-ProcessCopyOptions(ParseState *pstate,
- CopyFormatOptions *opts_out,
- bool is_from,
- List *options)
+ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out,
+ bool is_from, List **other_options, ParseState *pstate)
{
bool format_specified = false;
bool freeze_specified = false;
@@ -559,10 +557,6 @@ ProcessCopyOptions(ParseState *pstate,
bool reject_limit_specified = false;
ListCell *option;
- /* Support external use for option sanity checking */
- if (opts_out == NULL)
- opts_out = (CopyFormatOptions *) palloc0(sizeof(CopyFormatOptions));
-
opts_out->file_encoding = -1;
/* Extract options from the statement node tree */
@@ -583,6 +577,8 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->csv_mode = true;
else if (strcmp(fmt, "binary") == 0)
opts_out->binary = true;
+ else if (FindCustomCopyFormat(fmt))
+ /* just validate option value */ ;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -731,11 +727,7 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->reject_limit = defGetCopyRejectLimitOption(defel);
}
else
- ereport(ERROR,
- (errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("option \"%s\" not recognized",
- defel->defname),
- parser_errposition(pstate, defel->location)));
+ *other_options = lappend(*other_options, defel);
}
/*
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 592be4fcb5d..6ca5466f244 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1705,7 +1705,7 @@ BeginCopyFrom(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options);
+ ProcessCopyFromOptions(cstate, options, pstate);
/* Process the target relation */
cstate->rel = rel;
@@ -2028,3 +2028,32 @@ ClosePipeFromProgram(CopyFromState cstate)
errdetail_internal("%s", wait_result_to_str(pclose_rc))));
}
}
+
+void
+ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate)
+{
+ bool temp_state = false;
+ List *other_options = NIL;
+ CopyFormatOptions *opts;
+
+ if (cstate == NULL)
+ {
+ cstate = create_copyfrom_state(pstate, options);
+ temp_state = true;
+ }
+
+ opts = &cstate->opts;
+
+ ProcessCopyBuiltinOptions(options, opts, true, &other_options, pstate);
+
+ foreach_node(DefElem, option, other_options)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", option->defname),
+ parser_errposition(pstate, option->location)));
+ }
+
+ if (temp_state)
+ pfree(cstate);
+}
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b1b3ae141eb..ea31fa911f9 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -84,6 +84,7 @@ static void CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string
bool use_quote);
static void CopyRelationTo(CopyToState cstate, Relation rel, Relation root_rel,
uint64 *processed);
+static void ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate);
/* built-in format-specific routines */
static Size CopyToEstimateStateTextLike(void);
@@ -785,7 +786,7 @@ BeginCopyTo(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options);
+ ProcessCopyToOptions(cstate, options, pstate);
/* Process the source/target relation or query */
if (rel)
@@ -1570,3 +1571,32 @@ CreateCopyDestReceiver(void)
return (DestReceiver *) self;
}
+
+static void
+ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate)
+{
+ bool temp_state = false;
+ List *other_options = NIL;
+ CopyFormatOptions *opts;
+
+ if (cstate == NULL)
+ {
+ cstate = create_copyto_state(pstate, options);
+ temp_state = true;
+ }
+
+ opts = &cstate->opts;
+
+ ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate);
+
+ foreach_node(DefElem, option, options)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", option->defname),
+ parser_errposition(pstate, option->location)));
+ }
+
+ if (temp_state)
+ pfree(cstate);
+}
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index d2d19536151..f457e2bee50 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -100,6 +100,8 @@ typedef struct CopyFromStateBuiltins
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
+extern void ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate);
+
/* One-row callbacks for built-in formats defined in copyfromparse.c */
extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext,
Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
--
2.47.3
[application/octet-stream] 0005-Add-Copy-To-From-ProcessOneOption-callback.patch (5.9K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/5-0005-Add-Copy-To-From-ProcessOneOption-callback.patch)
download | inline diff:
From 819a96d8cac5d41eae1dd2b32cdfd817199dd5ad Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 10 Nov 2025 15:05:05 -0800
Subject: [PATCH 5/6] Add Copy{To,From}ProcessOneOption callback.
---
src/backend/commands/copyfrom.c | 15 +++++++++++----
src/backend/commands/copyto.c | 17 ++++++++++++-----
src/include/commands/copyapi.h | 18 +++++++++++++++++-
3 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 6ca5466f244..c23c4b1188a 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -132,6 +132,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate);
/* text format */
static const CopyFromRoutine CopyFromRoutineText = {
.CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
+ .CopyFromProcessOneOption = NULL,
.CopyFromInFunc = CopyFromTextLikeInFunc,
.CopyFromStart = CopyFromTextLikeStart,
.CopyFromOneRow = CopyFromTextOneRow,
@@ -141,6 +142,7 @@ static const CopyFromRoutine CopyFromRoutineText = {
/* CSV format */
static const CopyFromRoutine CopyFromRoutineCSV = {
.CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
+ .CopyFromProcessOneOption = NULL,
.CopyFromInFunc = CopyFromTextLikeInFunc,
.CopyFromStart = CopyFromTextLikeStart,
.CopyFromOneRow = CopyFromCSVOneRow,
@@ -150,6 +152,7 @@ static const CopyFromRoutine CopyFromRoutineCSV = {
/* binary format */
static const CopyFromRoutine CopyFromRoutineBinary = {
.CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
+ .CopyFromProcessOneOption = NULL,
.CopyFromInFunc = CopyFromBinaryInFunc,
.CopyFromStart = CopyFromBinaryStart,
.CopyFromOneRow = CopyFromBinaryOneRow,
@@ -2048,10 +2051,14 @@ ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate)
foreach_node(DefElem, option, other_options)
{
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format \"%s\" not recognized", option->defname),
- parser_errposition(pstate, option->location)));
+ if (cstate->routine->CopyFromProcessOneOption &&
+ cstate->routine->CopyFromProcessOneOption(cstate, option))
+ /* custom option is processed */ ;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", option->defname),
+ parser_errposition(pstate, option->location)));
}
if (temp_state)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index ea31fa911f9..9136098a0ed 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -122,6 +122,7 @@ static void CopySendInt16(CopyToState cstate, int16 val);
/* text format */
static const CopyToRoutine CopyToRoutineText = {
.CopyToEstimateStateSpace = CopyToEstimateStateTextLike,
+ .CopyToProcessOneOption = NULL,
.CopyToStart = CopyToTextLikeStart,
.CopyToOutFunc = CopyToTextLikeOutFunc,
.CopyToOneRow = CopyToTextOneRow,
@@ -131,6 +132,7 @@ static const CopyToRoutine CopyToRoutineText = {
/* CSV format */
static const CopyToRoutine CopyToRoutineCSV = {
.CopyToEstimateStateSpace = CopyToEstimateStateTextLike,
+ .CopyToProcessOneOption = NULL,
.CopyToStart = CopyToTextLikeStart,
.CopyToOutFunc = CopyToTextLikeOutFunc,
.CopyToOneRow = CopyToCSVOneRow,
@@ -140,6 +142,7 @@ static const CopyToRoutine CopyToRoutineCSV = {
/* binary format */
static const CopyToRoutine CopyToRoutineBinary = {
.CopyToEstimateStateSpace = CopyToEstimateStateBinary,
+ .CopyToProcessOneOption = NULL,
.CopyToStart = CopyToBinaryStart,
.CopyToOutFunc = CopyToBinaryOutFunc,
.CopyToOneRow = CopyToBinaryOneRow,
@@ -1589,12 +1592,16 @@ ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate)
ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate);
- foreach_node(DefElem, option, options)
+ foreach_node(DefElem, option, other_options)
{
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format \"%s\" not recognized", option->defname),
- parser_errposition(pstate, option->location)));
+ if (cstate->routine->CopyToProcessOneOption &&
+ cstate->routine->CopyToProcessOneOption(cstate, option))
+ /* custom option is processed */ ;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", option->defname),
+ parser_errposition(pstate, option->location)));
}
if (temp_state)
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 2090a4e1c61..e19279d0c3d 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -27,6 +27,14 @@ typedef struct CopyToRoutine
*/
Size (*CopyToEstimateStateSpace) (void);
+ /*
+ * Process one COPY TO option. Return true if the option is processed,
+ * otherwise return false.
+ *
+ * This is an optional callback.
+ */
+ bool (*CopyToProcessOneOption) (CopyToState cstate, DefElem *option);
+
/*
* Set output function information. This callback is called once at the
* beginning of COPY TO.
@@ -70,6 +78,14 @@ typedef struct CopyFromRoutine
*/
Size (*CopyFromEstimateStateSpace) (void);
+ /*
+ * Process one COPY FROM option. Return true if the option is processed,
+ * otherwise return false.
+ *
+ * This is an optional callback.
+ */
+ bool (*CopyFromProcessOneOption) (CopyFromState cstate, DefElem *option);
+
/*
* Set input function information. This callback is called once at the
* beginning of COPY FROM.
@@ -112,7 +128,7 @@ typedef struct CopyFromRoutine
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
-extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread);
+extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread);
extern void CopyToFlushData(CopyToState cstate);
extern void RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine,
--
2.47.3
[application/octet-stream] 0002-Separate-format-specific-fields-from-CopyFromStateDa.patch (55.2K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/6-0002-Separate-format-specific-fields-from-CopyFromStateDa.patch)
download | inline diff:
From 54a307c3932d9fb080fdec3ce8b1068138f35f24 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Fri, 7 Nov 2025 15:33:32 -0800
Subject: [PATCH 2/6] Separate format-specific fields from CopyFromStateData
struct.
Previously, CopyFromStateData struct contained all state variables
used throughout COPY FROM operations. To support the upcoming
pluggable COPY format feature, this commit moves fields specific to
built-in formats into their own dedicated struct.
This commit also introduces a new CopyFromRoutine callback named
CopyFromEstimateStateSpace, allowing format routines to estimate and
return the memory size needed for their state data. The format's
state data must embed CopyFromStateData as its first field, and the
returned size must be equal to or larger than
CopyFromStateData. While separate structs for core and
format-specific usage might seem feasible, using contiguous memory
avoids overhead from extra pointer traversal.
This commit mixed many changes but the main goal is to move some
fields that are used only for built-in formats from out of
CopyFromStateData so that format implementation can use their own
state data, similar to what we did for COPY TO. Here is the summary of
changes:
* Introduce a new callback CopyFromEstimateStateSpace.
* Introduce a new struct CopyFromStateBuiltins for built-in format
state data.
* CopyFromStateData has fields used by the core.
* The new struct embeds CopyFromStateData at the first field.
* line_buf, raw_buf, and attribute_buf and their related fields are
now format-specific fields.
* Move some initialization steps (e.g., encoding convesion
preparation) to format's CopyFromStart callback.
Callback refactoring/handling.
==============================
One concern is how to integrate the error callback with
custom format. Currently, COPY FROM's error callback
CopyFromErrorCallback() is set right before the main loop in
CopyFrom() for COPY FROM command and unset after the loop, whereas
COPY command API allows users to call just BeginCopyFrom(),
NextCopyFrom(), and EndCopyFrom() (i.e., without CopyFrom()). I
initially thought we can think that the error callback is
format-specific function so we can set/unset the callback within the
COPY format callbacks (e.g., setting the callback in CopyFromStart
callback etc). However, I found out that it's not straightforward for
example because CopyFromStart can be called multiple times by
file_fdw.c. The current idea is that we allows format-implementations
to update the core's cur_XXX fields so that the core's error callback
can print the details. IOW, the error callback function is still a
core function that allows format-implementation can access and
update. While it looks working, one concern is priting the input row;
now that the input buffer and line buffer are format-speicic fileds,
there is no way for the core to access them to print the actual line
being read. One trick I used is to have the format implementation set
a pointer in the core's CopyFromStateData to the line_buf in format's
CopyFromStateBuiltins.
---
contrib/file_fdw/file_fdw.c | 5 +-
src/backend/commands/copyfrom.c | 253 ++++++++++++--------
src/backend/commands/copyfromparse.c | 292 +++++++++++++----------
src/include/commands/copy.h | 12 +-
src/include/commands/copyapi.h | 8 +-
src/include/commands/copyfrom_internal.h | 104 +-------
src/include/commands/copystate.h | 97 ++++++++
7 files changed, 436 insertions(+), 335 deletions(-)
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 70564a68b13..af2d86e5b5c 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -766,7 +766,8 @@ retry:
*/
ExecClearTuple(slot);
- if (NextCopyFrom(cstate, econtext, slot->tts_values, slot->tts_isnull))
+ if (NextCopyFrom(cstate, econtext, slot->tts_values, slot->tts_isnull,
+ NULL))
{
if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE &&
cstate->escontext->error_occurred)
@@ -1246,7 +1247,7 @@ file_acquire_sample_rows(Relation onerel, int elevel,
MemoryContextReset(tupcontext);
MemoryContextSwitchTo(tupcontext);
- found = NextCopyFrom(cstate, NULL, values, nulls);
+ found = NextCopyFrom(cstate, NULL, values, nulls, NULL);
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 12781963b4f..0c51e5ba5e1 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -30,6 +30,8 @@
#include "catalog/namespace.h"
#include "commands/copyapi.h"
#include "commands/copyfrom_internal.h"
+#include "commands/copystate.h"
+#include "commands/defrem.h"
#include "commands/progress.h"
#include "commands/trigger.h"
#include "executor/execPartition.h"
@@ -129,6 +131,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate);
/* text format */
static const CopyFromRoutine CopyFromRoutineText = {
+ .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
.CopyFromInFunc = CopyFromTextLikeInFunc,
.CopyFromStart = CopyFromTextLikeStart,
.CopyFromOneRow = CopyFromTextOneRow,
@@ -137,6 +140,7 @@ static const CopyFromRoutine CopyFromRoutineText = {
/* CSV format */
static const CopyFromRoutine CopyFromRoutineCSV = {
+ .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
.CopyFromInFunc = CopyFromTextLikeInFunc,
.CopyFromStart = CopyFromTextLikeStart,
.CopyFromOneRow = CopyFromCSVOneRow,
@@ -145,54 +149,129 @@ static const CopyFromRoutine CopyFromRoutineCSV = {
/* binary format */
static const CopyFromRoutine CopyFromRoutineBinary = {
+ .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
.CopyFromInFunc = CopyFromBinaryInFunc,
.CopyFromStart = CopyFromBinaryStart,
.CopyFromOneRow = CopyFromBinaryOneRow,
.CopyFromEnd = CopyFromBinaryEnd,
};
-/* Return a COPY FROM routine for the given options */
-static const CopyFromRoutine *
-CopyFromGetRoutine(const CopyFormatOptions *opts)
+/*
+ * Common routine to initialize CopyFromStateBuiltins data.
+ */
+static void
+initialize_copyfrom_bultins_state(CopyFromStateBuiltins * state, TupleDesc tupDesc)
{
- if (opts->csv_mode)
- return &CopyFromRoutineCSV;
- else if (opts->binary)
- return &CopyFromRoutineBinary;
+ /* Use client encoding when ENCODING option is not specified. */
+ if (state->base.opts.file_encoding < 0)
+ state->file_encoding = pg_get_client_encoding();
+ else
+ state->file_encoding = state->base.opts.file_encoding;
+
+ /*
+ * Look up encoding conversion function.
+ */
+ if (state->file_encoding == GetDatabaseEncoding() ||
+ state->file_encoding == PG_SQL_ASCII ||
+ GetDatabaseEncoding() == PG_SQL_ASCII)
+ {
+ state->need_transcoding = false;
+ }
+ else
+ {
+ state->need_transcoding = true;
+ state->conversion_proc = FindDefaultConversionProc(state->file_encoding,
+ GetDatabaseEncoding());
+ if (!OidIsValid(state->conversion_proc))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_FUNCTION),
+ errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
+ pg_encoding_to_char(state->file_encoding),
+ pg_encoding_to_char(GetDatabaseEncoding()))));
+ }
- /* default is text */
- return &CopyFromRoutineText;
+ state->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool));
+
+ /* initialize variables */
+ state->eol_type = EOL_UNKNOWN;
+
+ /* Convert convert_selectively name list to per-column flags */
+ if (state->base.opts.convert_selectively)
+ {
+ List *attnums;
+ ListCell *cur;
+ int num_phys_attrs = RelationGetDescr(state->base.rel)->natts;
+
+ state->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
+
+ attnums = CopyGetAttnums(tupDesc, state->base.rel, state->base.opts.convert_select);
+
+ foreach(cur, attnums)
+ {
+ int attnum = lfirst_int(cur);
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ if (!list_member_int(state->base.attnumlist, attnum))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
+ errmsg_internal("selected column \"%s\" not referenced by COPY",
+ NameStr(attr->attname))));
+ state->convert_select_flags[attnum - 1] = true;
+ }
+ }
+
+ /*
+ * Allocate buffers for the input pipeline.
+ *
+ * attribute_buf and raw_buf are used in both text and binary modes, but
+ * input_buf and line_buf only in text mode.
+ */
+ state->raw_buf = palloc(RAW_BUF_SIZE + 1);
+ state->raw_buf_index = state->raw_buf_len = 0;
+
+ initStringInfo(&state->attribute_buf);
+
+ /* Initialize state variables */
+ state->base.cur_relname = RelationGetRelationName(state->base.rel);
+ state->base.cur_lineno = 0;
+ state->base.cur_attname = NULL;
+ state->base.cur_attval = NULL;
+ state->base.relname_only = false;
}
/* Implementation of the start callback for text and CSV formats */
static void
CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc)
{
+ CopyFromStateBuiltins *state = (CopyFromStateBuiltins *) cstate;
AttrNumber attr_count;
+ initialize_copyfrom_bultins_state(state, tupDesc);
+
/*
* If encoding conversion is needed, we need another buffer to hold the
* converted input data. Otherwise, we can just point input_buf to the
* same buffer as raw_buf.
*/
- if (cstate->need_transcoding)
+ if (state->need_transcoding)
{
- cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
- cstate->input_buf_index = cstate->input_buf_len = 0;
+ state->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
+ state->input_buf_index = state->input_buf_len = 0;
}
else
- cstate->input_buf = cstate->raw_buf;
- cstate->input_reached_eof = false;
+ state->input_buf = state->raw_buf;
+ state->input_reached_eof = false;
- initStringInfo(&cstate->line_buf);
+ initStringInfo(&state->line_buf);
+ state->base.line_buf = &state->line_buf;
/*
* Create workspace for CopyReadAttributes results; used by CSV and text
* format.
*/
- attr_count = list_length(cstate->attnumlist);
- cstate->max_fields = attr_count;
- cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
+ attr_count = list_length(state->base.attnumlist);
+ state->max_fields = attr_count;
+ state->raw_fields = (char **) palloc(attr_count * sizeof(char *));
}
/*
@@ -220,6 +299,8 @@ CopyFromTextLikeEnd(CopyFromState cstate)
static void
CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc)
{
+ initialize_copyfrom_bultins_state((CopyFromStateBuiltins *) cstate, tupDesc);
+
/* Read and verify binary header */
ReceiveCopyBinaryHeader(cstate);
}
@@ -308,7 +389,7 @@ CopyFromErrorCallback(void *arg)
{
char *lineval;
- lineval = CopyLimitPrintoutLength(cstate->line_buf.data);
+ lineval = CopyLimitPrintoutLength(cstate->line_buf->data);
errcontext("COPY %s, line %" PRIu64 ": \"%s\"",
cstate->cur_relname,
cstate->cur_lineno, lineval);
@@ -1113,6 +1194,7 @@ CopyFrom(CopyFromState cstate)
{
TupleTableSlot *myslot;
bool skip_tuple;
+ CopyFromRowInfo rowinfo = {0};
CHECK_FOR_INTERRUPTS();
@@ -1146,7 +1228,8 @@ CopyFrom(CopyFromState cstate)
ExecClearTuple(myslot);
/* Directly store the values/nulls array in the slot */
- if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
+ if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull,
+ &rowinfo))
break;
if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE &&
@@ -1379,8 +1462,8 @@ CopyFrom(CopyFromState cstate)
/* Add this tuple to the tuple buffer */
CopyMultiInsertInfoStore(&multiInsertInfo,
resultRelInfo, myslot,
- cstate->line_buf.len,
- cstate->cur_lineno);
+ rowinfo.tuplen,
+ rowinfo.lineno);
/*
* If enough inserts have queued up, then flush all
@@ -1512,6 +1595,50 @@ CopyFrom(CopyFromState cstate)
return processed;
}
+static CopyFromState
+create_copyfrom_state(ParseState *pstate, List *options)
+{
+ const CopyFromRoutine *routine;
+ CopyFromState cstate;
+ Size req_size;
+ bool format_specified = false;
+
+ routine = &CopyFromRoutineText; /* default */
+ foreach_node(DefElem, defel, options)
+ {
+ if (strcmp(defel->defname, "format") == 0)
+ {
+ char *fmt = defGetString(defel);
+
+ if (format_specified)
+ errorConflictingDefElem(defel, pstate);
+ format_specified = true;
+ if (strcmp(fmt, "text") == 0)
+ routine = &CopyFromRoutineText;
+ else if (strcmp(fmt, "csv") == 0)
+ routine = &CopyFromRoutineCSV;
+ else if (strcmp(fmt, "binary") == 0)
+ routine = &CopyFromRoutineBinary;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", fmt),
+ parser_errposition(pstate, defel->location)));
+
+ break;
+ }
+ }
+
+ req_size = routine->CopyFromEstimateStateSpace();
+ Assert(req_size >= sizeof(CopyFromStateData));
+
+ /* Allocate workspace and zero all fields */
+ cstate = (CopyFromState) palloc0(req_size);
+ cstate->routine = routine;
+
+ return cstate;
+}
+
/*
* Setup to read tuples from a file for COPY FROM.
*
@@ -1558,7 +1685,7 @@ BeginCopyFrom(ParseState *pstate,
};
/* Allocate workspace and zero all fields */
- cstate = (CopyFromStateData *) palloc0(sizeof(CopyFromStateData));
+ cstate = create_copyfrom_state(pstate, options);
/*
* We allocate everything used by a cstate in a new memory context. This
@@ -1573,9 +1700,6 @@ BeginCopyFrom(ParseState *pstate,
/* Extract options from the statement node tree */
ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options);
- /* Set the format routine */
- cstate->routine = CopyFromGetRoutine(&cstate->opts);
-
/* Process the target relation */
cstate->rel = rel;
@@ -1657,82 +1781,10 @@ BeginCopyFrom(ParseState *pstate,
}
}
- /* Convert convert_selectively name list to per-column flags */
- if (cstate->opts.convert_selectively)
- {
- List *attnums;
- ListCell *cur;
-
- cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
-
- attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.convert_select);
-
- foreach(cur, attnums)
- {
- int attnum = lfirst_int(cur);
- Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
-
- if (!list_member_int(cstate->attnumlist, attnum))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
- errmsg_internal("selected column \"%s\" not referenced by COPY",
- NameStr(attr->attname))));
- cstate->convert_select_flags[attnum - 1] = true;
- }
- }
-
- /* Use client encoding when ENCODING option is not specified. */
- if (cstate->opts.file_encoding < 0)
- cstate->file_encoding = pg_get_client_encoding();
- else
- cstate->file_encoding = cstate->opts.file_encoding;
-
- /*
- * Look up encoding conversion function.
- */
- if (cstate->file_encoding == GetDatabaseEncoding() ||
- cstate->file_encoding == PG_SQL_ASCII ||
- GetDatabaseEncoding() == PG_SQL_ASCII)
- {
- cstate->need_transcoding = false;
- }
- else
- {
- cstate->need_transcoding = true;
- cstate->conversion_proc = FindDefaultConversionProc(cstate->file_encoding,
- GetDatabaseEncoding());
- if (!OidIsValid(cstate->conversion_proc))
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_FUNCTION),
- errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
- pg_encoding_to_char(cstate->file_encoding),
- pg_encoding_to_char(GetDatabaseEncoding()))));
- }
-
cstate->copy_src = COPY_FILE; /* default */
cstate->whereClause = whereClause;
- /* Initialize state variables */
- cstate->eol_type = EOL_UNKNOWN;
- cstate->cur_relname = RelationGetRelationName(cstate->rel);
- cstate->cur_lineno = 0;
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
- cstate->relname_only = false;
-
- /*
- * Allocate buffers for the input pipeline.
- *
- * attribute_buf and raw_buf are used in both text and binary modes, but
- * input_buf and line_buf only in text mode.
- */
- cstate->raw_buf = palloc(RAW_BUF_SIZE + 1);
- cstate->raw_buf_index = cstate->raw_buf_len = 0;
- cstate->raw_reached_eof = false;
-
- initStringInfo(&cstate->attribute_buf);
-
/* Assign range table and rteperminfos, we'll need them in CopyFrom. */
if (pstate)
{
@@ -1818,8 +1870,6 @@ BeginCopyFrom(ParseState *pstate,
}
}
- cstate->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool));
-
/* initialize progress */
pgstat_progress_start_command(PROGRESS_COMMAND_COPY,
cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid);
@@ -1833,6 +1883,7 @@ BeginCopyFrom(ParseState *pstate,
cstate->volatile_defexprs = volatile_defexprs;
cstate->num_defaults = num_defaults;
cstate->is_program = is_program;
+ cstate->reached_eof = false;
if (data_source_cb)
{
@@ -1959,7 +2010,7 @@ ClosePipeFromProgram(CopyFromState cstate)
* should not report that as an error. Otherwise, SIGPIPE indicates a
* problem.
*/
- if (!cstate->raw_reached_eof &&
+ if (!cstate->reached_eof &&
wait_result_is_signal(pclose_rc, SIGPIPE))
return;
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index b1ae97b833d..669dbfb8459 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -138,21 +138,20 @@ if (1) \
/* NOTE: there's a copy of this in copyto.c */
static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
-
/* non-export function prototypes */
-static bool CopyReadLine(CopyFromState cstate, bool is_csv);
-static bool CopyReadLineText(CopyFromState cstate, bool is_csv);
-static int CopyReadAttributesText(CopyFromState cstate);
-static int CopyReadAttributesCSV(CopyFromState cstate);
-static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
+static bool CopyReadLine(CopyFromStateBuiltins * state, bool is_csv);
+static bool CopyReadLineText(CopyFromStateBuiltins * state, bool is_csv);
+static int CopyReadAttributesText(CopyFromStateBuiltins * state);
+static int CopyReadAttributesCSV(CopyFromStateBuiltins * state);
+static Datum CopyReadBinaryAttribute(CopyFromStateBuiltins * state, FmgrInfo *flinfo,
Oid typioparam, int32 typmod,
bool *isnull);
-static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate,
+static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromStateBuiltins * state,
ExprContext *econtext,
- Datum *values,
- bool *nulls,
+ Datum *values, bool *nulls,
+ CopyFromRowInfo * rowinfo,
bool is_csv);
-static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate,
+static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromStateBuiltins * state,
char ***fields,
int *nfields,
bool is_csv);
@@ -161,10 +160,10 @@ static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromSta
/* Low-level communications functions */
static int CopyGetData(CopyFromState cstate, void *databuf,
int minread, int maxread);
-static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
-static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
-static void CopyLoadInputBuf(CopyFromState cstate);
-static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
+static inline bool CopyGetInt32(CopyFromStateBuiltins * state, int32 *val);
+static inline bool CopyGetInt16(CopyFromStateBuiltins * state, int16 *val);
+static void CopyLoadInputBuf(CopyFromStateBuiltins * state);
+static int CopyReadBinaryData(CopyFromStateBuiltins * state, char *dest, int nbytes);
void
ReceiveCopyBegin(CopyFromState cstate)
@@ -187,8 +186,9 @@ ReceiveCopyBegin(CopyFromState cstate)
}
void
-ReceiveCopyBinaryHeader(CopyFromState cstate)
+ReceiveCopyBinaryHeader(CopyFromState ccstate)
{
+ CopyFromStateBuiltins *cstate = (CopyFromStateBuiltins *) ccstate;
char readSig[11];
int32 tmp;
@@ -255,10 +255,10 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
(errcode_for_file_access(),
errmsg("could not read from COPY file: %m")));
if (bytesread == 0)
- cstate->raw_reached_eof = true;
+ cstate->reached_eof = true;
break;
case COPY_FRONTEND:
- while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
+ while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
{
int avail;
@@ -309,7 +309,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
break;
case PqMsg_CopyDone:
/* COPY IN correctly terminated by frontend */
- cstate->raw_reached_eof = true;
+ cstate->reached_eof = true;
return bytesread;
case PqMsg_CopyFail:
ereport(ERROR,
@@ -359,7 +359,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
* Returns true if OK, false if EOF
*/
static inline bool
-CopyGetInt32(CopyFromState cstate, int32 *val)
+CopyGetInt32(CopyFromStateBuiltins * cstate, int32 *val)
{
uint32 buf;
@@ -376,7 +376,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
* CopyGetInt16 reads an int16 that appears in network byte order
*/
static inline bool
-CopyGetInt16(CopyFromState cstate, int16 *val)
+CopyGetInt16(CopyFromStateBuiltins * cstate, int16 *val)
{
uint16 buf;
@@ -397,7 +397,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
* On entry, there must be some data to convert in 'raw_buf'.
*/
static void
-CopyConvertBuf(CopyFromState cstate)
+CopyConvertBuf(CopyFromStateBuiltins * cstate)
{
/*
* If the file and server encoding are the same, no encoding conversion is
@@ -421,7 +421,7 @@ CopyConvertBuf(CopyFromState cstate)
/*
* If no more raw data is coming, report the EOF to the caller.
*/
- if (cstate->raw_reached_eof)
+ if (cstate->base.reached_eof)
cstate->input_reached_eof = true;
return;
}
@@ -444,7 +444,7 @@ CopyConvertBuf(CopyFromState cstate)
* least one character, and a failure to do so means that we've
* hit an invalid byte sequence.
*/
- if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
+ if (cstate->base.reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
cstate->input_reached_error = true;
return;
}
@@ -467,7 +467,7 @@ CopyConvertBuf(CopyFromState cstate)
/*
* If no more raw data is coming, report the EOF to the caller.
*/
- if (cstate->raw_reached_eof)
+ if (cstate->base.reached_eof)
cstate->input_reached_eof = true;
return;
}
@@ -517,7 +517,7 @@ CopyConvertBuf(CopyFromState cstate)
* failure to do so must mean that we've hit a byte sequence
* that's invalid.
*/
- if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
+ if (cstate->base.reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
cstate->input_reached_error = true;
return;
}
@@ -530,7 +530,7 @@ CopyConvertBuf(CopyFromState cstate)
* Report an encoding or conversion error.
*/
static void
-CopyConversionError(CopyFromState cstate)
+CopyConversionError(CopyFromStateBuiltins * cstate)
{
Assert(cstate->raw_buf_len > 0);
Assert(cstate->input_reached_error);
@@ -587,7 +587,7 @@ CopyConversionError(CopyFromState cstate)
* beginning of the buffer, and we load new data after that.
*/
static void
-CopyLoadRawBuf(CopyFromState cstate)
+CopyLoadRawBuf(CopyFromStateBuiltins * cstate)
{
int nbytes;
int inbytes;
@@ -624,17 +624,17 @@ CopyLoadRawBuf(CopyFromState cstate)
}
/* Load more data */
- inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
+ inbytes = CopyGetData((CopyFromState) cstate, cstate->raw_buf + cstate->raw_buf_len,
1, RAW_BUF_SIZE - cstate->raw_buf_len);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_len = nbytes;
- cstate->bytes_processed += inbytes;
- pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
+ cstate->base.bytes_processed += inbytes;
+ pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->base.bytes_processed);
if (inbytes == 0)
- cstate->raw_reached_eof = true;
+ cstate->base.reached_eof = true;
}
/*
@@ -647,7 +647,7 @@ CopyLoadRawBuf(CopyFromState cstate)
* of the buffer and then we load more data after that.
*/
static void
-CopyLoadInputBuf(CopyFromState cstate)
+CopyLoadInputBuf(CopyFromStateBuiltins * cstate)
{
int nbytes = INPUT_BUF_BYTES(cstate);
@@ -685,7 +685,7 @@ CopyLoadInputBuf(CopyFromState cstate)
break;
/* Try to load more raw data */
- Assert(!cstate->raw_reached_eof);
+ Assert(!cstate->base.reached_eof);
CopyLoadRawBuf(cstate);
}
}
@@ -698,7 +698,7 @@ CopyLoadInputBuf(CopyFromState cstate)
* would be less than 'nbytes' only if we reach EOF).
*/
static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+CopyReadBinaryData(CopyFromStateBuiltins * cstate, char *dest, int nbytes)
{
int copied_bytes = 0;
@@ -723,7 +723,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
if (RAW_BUF_BYTES(cstate) == 0)
{
CopyLoadRawBuf(cstate);
- if (cstate->raw_reached_eof)
+ if (cstate->base.reached_eof)
break; /* EOF */
}
@@ -739,15 +739,21 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
return copied_bytes;
}
+Size
+CopyFromBuiltinsEstimateSpace(void)
+{
+ return sizeof(CopyFromStateBuiltins);
+}
+
/*
* This function is exposed for use by extensions that read raw fields in the
* next line. See NextCopyFromRawFieldsInternal() for details.
*/
bool
-NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
+NextCopyFromRawFields(CopyFromState ccstate, char ***fields, int *nfields)
{
- return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
- cstate->opts.csv_mode);
+ return NextCopyFromRawFieldsInternal((CopyFromStateBuiltins *) ccstate,
+ fields, nfields, ccstate->opts.csv_mode);
}
/*
@@ -768,35 +774,35 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
* by internal functions such as CopyFromTextLikeOneRow().
*/
static pg_attribute_always_inline bool
-NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
+NextCopyFromRawFieldsInternal(CopyFromStateBuiltins * cstate, char ***fields, int *nfields, bool is_csv)
{
int fldct;
bool done = false;
/* only available for text or csv input */
- Assert(!cstate->opts.binary);
+ Assert(!cstate->base.opts.binary);
/* on input check that the header line is correct if needed */
- if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
+ if (cstate->base.cur_lineno == 0 && cstate->base.opts.header_line != COPY_HEADER_FALSE)
{
ListCell *cur;
TupleDesc tupDesc;
- int lines_to_skip = cstate->opts.header_line;
+ int lines_to_skip = cstate->base.opts.header_line;
/* If set to "match", one header line is skipped */
- if (cstate->opts.header_line == COPY_HEADER_MATCH)
+ if (cstate->base.opts.header_line == COPY_HEADER_MATCH)
lines_to_skip = 1;
- tupDesc = RelationGetDescr(cstate->rel);
+ tupDesc = RelationGetDescr(cstate->base.rel);
for (int i = 0; i < lines_to_skip; i++)
{
- cstate->cur_lineno++;
+ cstate->base.cur_lineno++;
if ((done = CopyReadLine(cstate, is_csv)))
break;
}
- if (cstate->opts.header_line == COPY_HEADER_MATCH)
+ if (cstate->base.opts.header_line == COPY_HEADER_MATCH)
{
int fldnum;
@@ -805,14 +811,14 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
else
fldct = CopyReadAttributesText(cstate);
- if (fldct != list_length(cstate->attnumlist))
+ if (fldct != list_length(cstate->base.attnumlist))
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("wrong number of fields in header line: got %d, expected %d",
- fldct, list_length(cstate->attnumlist))));
+ fldct, list_length(cstate->base.attnumlist))));
fldnum = 0;
- foreach(cur, cstate->attnumlist)
+ foreach(cur, cstate->base.attnumlist)
{
int attnum = lfirst_int(cur);
char *colName;
@@ -825,7 +831,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
- fldnum, cstate->opts.null_print, NameStr(attr->attname))));
+ fldnum, cstate->base.opts.null_print, NameStr(attr->attname))));
if (namestrcmp(&attr->attname, colName) != 0)
{
@@ -841,7 +847,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
return false;
}
- cstate->cur_lineno++;
+ cstate->base.cur_lineno++;
/* Actually read the line into memory here */
done = CopyReadLine(cstate, is_csv);
@@ -877,8 +883,8 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields
* relation passed to BeginCopyFrom. This function fills the arrays.
*/
bool
-NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls)
+NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Datum *values,
+ bool *nulls, CopyFromRowInfo * rowinfo)
{
TupleDesc tupDesc;
AttrNumber num_phys_attrs,
@@ -893,10 +899,9 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
/* Get one row from source */
- if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
+ if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls, rowinfo))
return false;
/*
@@ -923,17 +928,19 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
/* Implementation of the per-row callback for text format */
bool
CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
- bool *nulls)
+ bool *nulls, CopyFromRowInfo * rowinfo)
{
- return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false);
+ return CopyFromTextLikeOneRow((CopyFromStateBuiltins *) cstate, econtext,
+ values, nulls, rowinfo, false);
}
/* Implementation of the per-row callback for CSV format */
bool
CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
- bool *nulls)
+ bool *nulls, CopyFromRowInfo * rowinfo)
{
- return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
+ return CopyFromTextLikeOneRow((CopyFromStateBuiltins *) cstate, econtext,
+ values, nulls, rowinfo, true);
}
/*
@@ -943,22 +950,25 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
* and to help compilers to optimize away the 'is_csv' condition.
*/
static pg_attribute_always_inline bool
-CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls, bool is_csv)
+CopyFromTextLikeOneRow(CopyFromStateBuiltins * cstate, ExprContext *econtext,
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo,
+ bool is_csv)
{
TupleDesc tupDesc;
AttrNumber attr_count;
- FmgrInfo *in_functions = cstate->in_functions;
- Oid *typioparams = cstate->typioparams;
- ExprState **defexprs = cstate->defexprs;
+ FmgrInfo *in_functions = cstate->base.in_functions;
+ Oid *typioparams = cstate->base.typioparams;
+ ExprState **defexprs = cstate->base.defexprs;
char **field_strings;
ListCell *cur;
int fldct;
int fieldno;
char *string;
- tupDesc = RelationGetDescr(cstate->rel);
- attr_count = list_length(cstate->attnumlist);
+ tupDesc = RelationGetDescr(cstate->base.rel);
+ attr_count = list_length(cstate->base.attnumlist);
+
+ MemSet(cstate->defaults, false, tupDesc->natts * sizeof(bool));
/* read raw fields in the next line */
if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv))
@@ -973,7 +983,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
fieldno = 0;
/* Loop to read the user attributes on the line. */
- foreach(cur, cstate->attnumlist)
+ foreach(cur, cstate->base.attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
@@ -996,16 +1006,16 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
if (is_csv)
{
if (string == NULL &&
- cstate->opts.force_notnull_flags[m])
+ cstate->base.opts.force_notnull_flags[m])
{
/*
* FORCE_NOT_NULL option is set and column is NULL - convert
* it to the NULL string.
*/
- string = cstate->opts.null_print;
+ string = cstate->base.opts.null_print;
}
- else if (string != NULL && cstate->opts.force_null_flags[m]
- && strcmp(string, cstate->opts.null_print) == 0)
+ else if (string != NULL && cstate->base.opts.force_null_flags[m]
+ && strcmp(string, cstate->base.opts.null_print) == 0)
{
/*
* FORCE_NULL option is set and column matches the NULL
@@ -1017,8 +1027,8 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
}
}
- cstate->cur_attname = NameStr(att->attname);
- cstate->cur_attval = string;
+ cstate->base.cur_attname = NameStr(att->attname);
+ cstate->base.cur_attval = string;
if (string != NULL)
nulls[m] = false;
@@ -1039,73 +1049,81 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
string,
typioparams[m],
att->atttypmod,
- (Node *) cstate->escontext,
+ (Node *) cstate->base.escontext,
&values[m]))
{
- Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
+ Assert(cstate->base.opts.on_error != COPY_ON_ERROR_STOP);
- cstate->num_errors++;
+ cstate->base.num_errors++;
- if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
+ if (cstate->base.opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
{
/*
* Since we emit line number and column info in the below
* notice message, we suppress error context information other
* than the relation name.
*/
- Assert(!cstate->relname_only);
- cstate->relname_only = true;
+ Assert(!cstate->base.relname_only);
+ cstate->base.relname_only = true;
- if (cstate->cur_attval)
+ if (cstate->base.cur_attval)
{
char *attval;
- attval = CopyLimitPrintoutLength(cstate->cur_attval);
+ attval = CopyLimitPrintoutLength(cstate->base.cur_attval);
ereport(NOTICE,
errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
- cstate->cur_lineno,
- cstate->cur_attname,
+ cstate->base.cur_lineno,
+ cstate->base.cur_attname,
attval));
pfree(attval);
}
else
ereport(NOTICE,
errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
- cstate->cur_lineno,
- cstate->cur_attname));
+ cstate->base.cur_lineno,
+ cstate->base.cur_attname));
/* reset relname_only */
- cstate->relname_only = false;
+ cstate->base.relname_only = false;
}
return true;
}
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
+ cstate->base.cur_attname = NULL;
+ cstate->base.cur_attval = NULL;
}
Assert(fieldno == attr_count);
+ /* Set output parameters */
+ if (rowinfo)
+ {
+ rowinfo->lineno = cstate->base.cur_lineno;
+ rowinfo->tuplen = cstate->line_buf.len;
+ }
+
return true;
}
/* Implementation of the per-row callback for binary format */
bool
-CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
- bool *nulls)
+CopyFromBinaryOneRow(CopyFromState ccstate, ExprContext *econtext, Datum *values,
+ bool *nulls, CopyFromRowInfo * rowinfo)
{
+ CopyFromStateBuiltins *cstate = (CopyFromStateBuiltins *) ccstate;
TupleDesc tupDesc;
AttrNumber attr_count;
- FmgrInfo *in_functions = cstate->in_functions;
- Oid *typioparams = cstate->typioparams;
+ FmgrInfo *in_functions = cstate->base.in_functions;
+ Oid *typioparams = cstate->base.typioparams;
int16 fld_count;
ListCell *cur;
- tupDesc = RelationGetDescr(cstate->rel);
- attr_count = list_length(cstate->attnumlist);
+ tupDesc = RelationGetDescr(cstate->base.rel);
+ attr_count = list_length(cstate->base.attnumlist);
- cstate->cur_lineno++;
+ cstate->base.cur_lineno++;
if (!CopyGetInt16(cstate, &fld_count))
{
@@ -1138,19 +1156,29 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
errmsg("row field count is %d, expected %d",
(int) fld_count, attr_count)));
- foreach(cur, cstate->attnumlist)
+ foreach(cur, cstate->base.attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
- cstate->cur_attname = NameStr(att->attname);
+ cstate->base.cur_attname = NameStr(att->attname);
values[m] = CopyReadBinaryAttribute(cstate,
&in_functions[m],
typioparams[m],
att->atttypmod,
&nulls[m]);
- cstate->cur_attname = NULL;
+ cstate->base.cur_attname = NULL;
+ }
+
+ if (rowinfo)
+ {
+ /*
+ * XXX: We used to use line_buf.len but we don't actually use line_buf
+ * in binary format.
+ */
+ rowinfo->lineno = cstate->base.cur_lineno;
+ rowinfo->tuplen = cstate->line_buf.len;
}
return true;
@@ -1164,12 +1192,12 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
* in the final value of line_buf.
*/
static bool
-CopyReadLine(CopyFromState cstate, bool is_csv)
+CopyReadLine(CopyFromStateBuiltins * cstate, bool is_csv)
{
bool result;
resetStringInfo(&cstate->line_buf);
- cstate->line_buf_valid = false;
+ cstate->base.line_buf_valid = false;
/* Parse data and transfer into line_buf */
result = CopyReadLineText(cstate, is_csv);
@@ -1181,13 +1209,13 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (cstate->base.copy_src == COPY_FRONTEND)
{
int inbytes;
do
{
- inbytes = CopyGetData(cstate, cstate->input_buf,
+ inbytes = CopyGetData((CopyFromState) cstate, cstate->input_buf,
1, INPUT_BUF_SIZE);
} while (inbytes > 0);
cstate->input_buf_index = 0;
@@ -1231,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
}
/* Now it's safe to use the buffer in error messages */
- cstate->line_buf_valid = true;
+ cstate->base.line_buf_valid = true;
return result;
}
@@ -1240,7 +1268,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* CopyReadLineText - inner loop of CopyReadLine for text mode
*/
static bool
-CopyReadLineText(CopyFromState cstate, bool is_csv)
+CopyReadLineText(CopyFromStateBuiltins * cstate, bool is_csv)
{
char *copy_input_buf;
int input_buf_ptr;
@@ -1257,8 +1285,8 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
if (is_csv)
{
- quotec = cstate->opts.quote[0];
- escapec = cstate->opts.escape[0];
+ quotec = cstate->base.opts.quote[0];
+ escapec = cstate->base.opts.escape[0];
/* ignore special escape processing if it's the same as quotec */
if (quotec == escapec)
escapec = '\0';
@@ -1367,7 +1395,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv)
* at all --- is cur_lineno a physical or logical count?)
*/
if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
- cstate->cur_lineno++;
+ cstate->base.cur_lineno++;
}
/* Process \r */
@@ -1570,9 +1598,9 @@ GetDecimalFromHex(char hex)
* The return value is the number of fields actually read.
*/
static int
-CopyReadAttributesText(CopyFromState cstate)
+CopyReadAttributesText(CopyFromStateBuiltins * cstate)
{
- char delimc = cstate->opts.delim[0];
+ char delimc = cstate->base.opts.delim[0];
int fieldno;
char *output_ptr;
char *cur_ptr;
@@ -1755,26 +1783,26 @@ CopyReadAttributesText(CopyFromState cstate)
/* Check whether raw input matched null marker */
input_len = end_ptr - start_ptr;
- if (input_len == cstate->opts.null_print_len &&
- strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
+ if (input_len == cstate->base.opts.null_print_len &&
+ strncmp(start_ptr, cstate->base.opts.null_print, input_len) == 0)
cstate->raw_fields[fieldno] = NULL;
/* Check whether raw input matched default marker */
- else if (fieldno < list_length(cstate->attnumlist) &&
- cstate->opts.default_print &&
- input_len == cstate->opts.default_print_len &&
- strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
+ else if (fieldno < list_length(cstate->base.attnumlist) &&
+ cstate->base.opts.default_print &&
+ input_len == cstate->base.opts.default_print_len &&
+ strncmp(start_ptr, cstate->base.opts.default_print, input_len) == 0)
{
/* fieldno is 0-indexed and attnum is 1-indexed */
- int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
+ int m = list_nth_int(cstate->base.attnumlist, fieldno) - 1;
- if (cstate->defexprs[m] != NULL)
+ if (cstate->base.defexprs[m] != NULL)
{
/* defaults contain entries for all physical attributes */
cstate->defaults[m] = true;
}
else
{
- TupleDesc tupDesc = RelationGetDescr(cstate->rel);
+ TupleDesc tupDesc = RelationGetDescr(cstate->base.rel);
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
ereport(ERROR,
@@ -1824,11 +1852,11 @@ CopyReadAttributesText(CopyFromState cstate)
* "standard" (i.e. common) CSV usage.
*/
static int
-CopyReadAttributesCSV(CopyFromState cstate)
+CopyReadAttributesCSV(CopyFromStateBuiltins * cstate)
{
- char delimc = cstate->opts.delim[0];
- char quotec = cstate->opts.quote[0];
- char escapec = cstate->opts.escape[0];
+ char delimc = cstate->base.opts.delim[0];
+ char quotec = cstate->base.opts.quote[0];
+ char escapec = cstate->base.opts.escape[0];
int fieldno;
char *output_ptr;
char *cur_ptr;
@@ -1970,26 +1998,26 @@ endfield:
/* Check whether raw input matched null marker */
input_len = end_ptr - start_ptr;
- if (!saw_quote && input_len == cstate->opts.null_print_len &&
- strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
+ if (!saw_quote && input_len == cstate->base.opts.null_print_len &&
+ strncmp(start_ptr, cstate->base.opts.null_print, input_len) == 0)
cstate->raw_fields[fieldno] = NULL;
/* Check whether raw input matched default marker */
- else if (fieldno < list_length(cstate->attnumlist) &&
- cstate->opts.default_print &&
- input_len == cstate->opts.default_print_len &&
- strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
+ else if (fieldno < list_length(cstate->base.attnumlist) &&
+ cstate->base.opts.default_print &&
+ input_len == cstate->base.opts.default_print_len &&
+ strncmp(start_ptr, cstate->base.opts.default_print, input_len) == 0)
{
/* fieldno is 0-index and attnum is 1-index */
- int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
+ int m = list_nth_int(cstate->base.attnumlist, fieldno) - 1;
- if (cstate->defexprs[m] != NULL)
+ if (cstate->base.defexprs[m] != NULL)
{
/* defaults contain entries for all physical attributes */
cstate->defaults[m] = true;
}
else
{
- TupleDesc tupDesc = RelationGetDescr(cstate->rel);
+ TupleDesc tupDesc = RelationGetDescr(cstate->base.rel);
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
ereport(ERROR,
@@ -2019,7 +2047,7 @@ endfield:
* Read a binary attribute
*/
static Datum
-CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
+CopyReadBinaryAttribute(CopyFromStateBuiltins * cstate, FmgrInfo *flinfo,
Oid typioparam, int32 typmod,
bool *isnull)
{
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index d75a70715a4..30a1d2bff6e 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -48,6 +48,12 @@ typedef enum CopyLogVerbosityChoice
COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */
} CopyLogVerbosityChoice;
+typedef struct CopyFromRowInfo
+{
+ uint64 lineno;
+ int tuplen;
+} CopyFromRowInfo;
+
/*
* A struct to hold COPY options, in a parsed form. All of these are related
* to formatting, except for 'freeze', which doesn't really belong here, but
@@ -89,8 +95,6 @@ typedef struct CopyFormatOptions
/* defined in copystate.h */
typedef struct CopyToStateData *CopyToState;
-
-/* Private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
@@ -105,8 +109,8 @@ extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *where
const char *filename,
bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options);
extern void EndCopyFrom(CopyFromState cstate);
-extern bool NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+extern bool NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Datum *values,
+ bool *nulls, CopyFromRowInfo * rowinfo);
extern bool NextCopyFromRawFields(CopyFromState cstate,
char ***fields, int *nfields);
extern void CopyFromErrorCallback(void *arg);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 3b9982d54b8..c3d2199a0b6 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -65,6 +65,11 @@ typedef struct CopyToRoutine
*/
typedef struct CopyFromRoutine
{
+ /*
+ * Estimate and return the memory size required to store the state data.
+ */
+ Size (*CopyFromEstimateStateSpace) (void);
+
/*
* Set input function information. This callback is called once at the
* beginning of COPY FROM.
@@ -99,12 +104,11 @@ typedef struct CopyFromRoutine
* Returns false if there are no more tuples to read.
*/
bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
/*
* End a COPY FROM. This callback is called once at the end of COPY FROM.
*/
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
-
#endif /* COPYAPI_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index c8b22af22d8..d2d19536151 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -15,105 +15,23 @@
#define COPYFROM_INTERNAL_H
#include "commands/copy.h"
-#include "commands/trigger.h"
-#include "nodes/miscnodes.h"
+#include "commands/copystate.h"
/*
- * Represents the different source cases we need to worry about at
- * the bottom level
+ * COPY FROM state data used for builtin formats.
*/
-typedef enum CopySource
+typedef struct CopyFromStateBuiltins
{
- COPY_FILE, /* from file (or a piped program) */
- COPY_FRONTEND, /* from frontend */
- COPY_CALLBACK, /* from callback function */
-} CopySource;
-
-/*
- * Represents the end-of-line terminator type of the input
- */
-typedef enum EolType
-{
- EOL_UNKNOWN,
- EOL_NL,
- EOL_CR,
- EOL_CRNL,
-} EolType;
-
-/*
- * Represents the insert method to be used during COPY FROM.
- */
-typedef enum CopyInsertMethod
-{
- CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */
- CIM_MULTI, /* always use table_multi_insert or
- * ExecForeignBatchInsert */
- CIM_MULTI_CONDITIONAL, /* use table_multi_insert or
- * ExecForeignBatchInsert only if valid */
-} CopyInsertMethod;
-
-/*
- * This struct contains all the state variables used throughout a COPY FROM
- * operation.
- */
-typedef struct CopyFromStateData
-{
- /* format routine */
- const struct CopyFromRoutine *routine;
-
- /* low-level state data */
- CopySource copy_src; /* type of copy source */
- FILE *copy_file; /* used if copy_src == COPY_FILE */
- StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
+ CopyFromStateData base;
EolType eol_type; /* EOL type of input */
int file_encoding; /* file or remote side's character encoding */
bool need_transcoding; /* file encoding diff from server? */
Oid conversion_proc; /* encoding conversion function */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /* these are just for error messages, see CopyFromErrorCallback */
- const char *cur_relname; /* table name for error messages */
- uint64 cur_lineno; /* line number for error messages */
- const char *cur_attname; /* current att for error messages */
- const char *cur_attval; /* current att value for error messages */
- bool relname_only; /* don't output line number, att, etc. */
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- AttrNumber num_defaults; /* count of att that are missing and have
- * default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
- ErrorSaveContext *escontext; /* soft error trapped during in_functions
- * execution */
- uint64 num_errors; /* total number of rows which contained soft
- * errors */
- int *defmap; /* array of default att numbers related to
- * missing att */
- ExprState **defexprs; /* array of default att expressions for all
- * att */
bool *defaults; /* if DEFAULT marker was found for
* corresponding att */
- bool volatile_defexprs; /* is any of defexprs volatile? */
- List *range_table; /* single element list of RangeTblEntry */
- List *rteperminfos; /* single element list of RTEPermissionInfo */
- ExprState *qualexpr;
-
- TransitionCaptureState *transition_capture;
/*
* These variables are used to reduce overhead in COPY FROM.
@@ -141,7 +59,6 @@ typedef struct CopyFromStateData
* appropriate. (In binary mode, line_buf is not used.)
*/
StringInfoData line_buf;
- bool line_buf_valid; /* contains the row being processed? */
/*
* input_buf holds input data, already converted to database encoding.
@@ -175,23 +92,22 @@ typedef struct CopyFromStateData
char *raw_buf;
int raw_buf_index; /* next byte to process */
int raw_buf_len; /* total # of bytes stored */
- bool raw_reached_eof; /* true if we reached EOF */
/* Shorthand for number of unconsumed bytes available in raw_buf */
#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
-
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
+} CopyFromStateBuiltins;
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
/* One-row callbacks for built-in formats defined in copyfromparse.c */
extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
extern bool CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
+
+extern Size CopyFromBuiltinsEstimateSpace(void);
#endif /* COPYFROM_INTERNAL_H */
diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h
index 7561083a323..145dccd0f8f 100644
--- a/src/include/commands/copystate.h
+++ b/src/include/commands/copystate.h
@@ -17,6 +17,8 @@
#include "postgres.h"
#include "commands/copy.h"
#include "executor/execdesc.h"
+#include "commands/trigger.h"
+#include "nodes/miscnodes.h"
/*
* Represents the different dest cases we need to worry about at
@@ -65,4 +67,99 @@ typedef struct CopyToStateData
uint64 bytes_processed; /* number of bytes processed so far */
} CopyToStateData;
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_FILE, /* from file (or a piped program) */
+ COPY_FRONTEND, /* from frontend */
+ COPY_CALLBACK, /* from callback function */
+} CopySource;
+
+/*
+ * Represents the end-of-line terminator type of the input
+ */
+typedef enum EolType
+{
+ EOL_UNKNOWN,
+ EOL_NL,
+ EOL_CR,
+ EOL_CRNL,
+} EolType;
+
+/*
+ * Represents the insert method to be used during COPY FROM.
+ */
+typedef enum CopyInsertMethod
+{
+ CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */
+ CIM_MULTI, /* always use table_multi_insert or
+ * ExecForeignBatchInsert */
+ CIM_MULTI_CONDITIONAL, /* use table_multi_insert or
+ * ExecForeignBatchInsert only if valid */
+} CopyInsertMethod;
+
+/*
+ * This struct contains all the state variables used throughout a COPY FROM
+ * operation.
+ */
+typedef struct CopyFromStateData
+{
+ /* format routine */
+ const struct CopyFromRoutine *routine;
+
+ /* low-level state data */
+ CopySource copy_src; /* type of copy source */
+ FILE *copy_file; /* used if copy_src == COPY_FILE */
+ StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
+ bool reached_eof; /* true if we reached EOF */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /* these are just for error messages, see CopyFromErrorCallback */
+ const char *cur_relname; /* table name for error messages */
+ uint64 cur_lineno; /* line number for error messages */
+ const char *cur_attname; /* current att for error messages */
+ const char *cur_attval; /* current att value for error messages */
+ bool relname_only; /* don't output line number, att, etc. */
+ StringInfo line_buf;
+ bool line_buf_valid;
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+ ErrorSaveContext *escontext; /* soft error trapped during in_functions
+ * execution */
+ uint64 num_errors; /* total number of rows which contained soft
+ * errors */
+ AttrNumber num_defaults; /* count of att that are missing and have
+ * default value */
+ int *defmap; /* array of default att numbers related to
+ * missing att */
+ ExprState **defexprs; /* array of default att expressions for all
+ * att */
+ bool volatile_defexprs; /* is any of defexprs volatile? */
+ List *range_table; /* single element list of RangeTblEntry */
+ List *rteperminfos; /* single element list of RTEPermissionInfo */
+ ExprState *qualexpr;
+
+ TransitionCaptureState *transition_capture;
+
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyFromStateData;
+
#endif
--
2.47.3
[application/octet-stream] 0001-Separate-format-specific-fields-from-CopyToStateData.patch (21.5K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/7-0001-Separate-format-specific-fields-from-CopyToStateData.patch)
download | inline diff:
From 7d8ac622144e20c9e3aaf958bc0644a3c74fffea Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Thu, 6 Nov 2025 00:02:03 -0800
Subject: [PATCH 1/6] Separate format-specific fields from CopyToStateData
struct.
Previously, CopyToStateData struct contained all state variables used
throughout COPY TO operations. To support the upcoming pluggable COPY
format feature, this commit moves fields specific to built-in formats
into their own dedicated struct.
This commit also introduces a new callback CopyToEstimateStateSpace,
allowing format routines to estimate and return the memory size needed
for their state data. The format's state data must embed
CopyToStateData as its first field, and the returned size must be
equal to or larger than CopyToStateData. While separate structs for
core and format-specific usage might seem feasible, using
contiguousmemory avoids overhead from extra pointer traversal.
---
src/backend/commands/copyto.c | 249 ++++++++++++++++---------------
src/include/commands/copy.h | 6 +-
src/include/commands/copyapi.h | 5 +
src/include/commands/copystate.h | 68 +++++++++
4 files changed, 208 insertions(+), 120 deletions(-)
create mode 100644 src/include/commands/copystate.h
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cef452584e5..6a0a66507ba 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -22,6 +22,8 @@
#include "access/tableam.h"
#include "catalog/pg_inherits.h"
#include "commands/copyapi.h"
+#include "commands/copystate.h"
+#include "commands/defrem.h"
#include "commands/progress.h"
#include "executor/execdesc.h"
#include "executor/executor.h"
@@ -39,19 +41,7 @@
#include "utils/snapmgr.h"
/*
- * Represents the different dest cases we need to worry about at
- * the bottom level
- */
-typedef enum CopyDest
-{
- COPY_FILE, /* to file (or a piped program) */
- COPY_FRONTEND, /* to frontend */
- COPY_CALLBACK, /* to callback function */
-} CopyDest;
-
-/*
- * This struct contains all the state variables used throughout a COPY TO
- * operation.
+ * Struct used for text and CSV format.
*
* Multi-byte encodings: all supported client-side encodings encode multi-byte
* characters by having the first byte's high bit set. Subsequent bytes of the
@@ -64,41 +54,14 @@ typedef enum CopyDest
* invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
* when we have to do it the hard way.
*/
-typedef struct CopyToStateData
+typedef struct CopyToStateTextLike
{
- /* format-specific routines */
- const CopyToRoutine *routine;
-
- /* low-level state data */
- CopyDest copy_dest; /* type of copy source/destination */
- FILE *copy_file; /* used if copy_dest == COPY_FILE */
- StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+ CopyToStateData base;
int file_encoding; /* file or remote side's character encoding */
bool need_transcoding; /* file encoding diff from server? */
bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
- List *partitions; /* OID list of partitions to copy data from */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- FmgrInfo *out_functions; /* lookup info for output functions */
- MemoryContext rowcontext; /* per-row evaluation context */
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
+} CopyToStateTextLike;
/* DestReceiver for COPY (query) TO */
typedef struct
@@ -116,13 +79,14 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
static void EndCopy(CopyToState cstate);
static void ClosePipeToProgram(CopyToState cstate);
static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot);
-static void CopyAttributeOutText(CopyToState cstate, const char *string);
-static void CopyAttributeOutCSV(CopyToState cstate, const char *string,
+static void CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string);
+static void CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string,
bool use_quote);
static void CopyRelationTo(CopyToState cstate, Relation rel, Relation root_rel,
uint64 *processed);
/* built-in format-specific routines */
+static Size CopyToEstimateStateTextLike(void);
static void CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc);
static void CopyToTextLikeOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo);
static void CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot);
@@ -130,6 +94,7 @@ static void CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot);
static void CopyToTextLikeOneRow(CopyToState cstate, TupleTableSlot *slot,
bool is_csv);
static void CopyToTextLikeEnd(CopyToState cstate);
+static Size CopyToEstimateStateBinary(void);
static void CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc);
static void CopyToBinaryOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo);
static void CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot);
@@ -155,6 +120,7 @@ static void CopySendInt16(CopyToState cstate, int16 val);
/* text format */
static const CopyToRoutine CopyToRoutineText = {
+ .CopyToEstimateStateSpace = CopyToEstimateStateTextLike,
.CopyToStart = CopyToTextLikeStart,
.CopyToOutFunc = CopyToTextLikeOutFunc,
.CopyToOneRow = CopyToTextOneRow,
@@ -163,6 +129,7 @@ static const CopyToRoutine CopyToRoutineText = {
/* CSV format */
static const CopyToRoutine CopyToRoutineCSV = {
+ .CopyToEstimateStateSpace = CopyToEstimateStateTextLike,
.CopyToStart = CopyToTextLikeStart,
.CopyToOutFunc = CopyToTextLikeOutFunc,
.CopyToOneRow = CopyToCSVOneRow,
@@ -171,62 +138,77 @@ static const CopyToRoutine CopyToRoutineCSV = {
/* binary format */
static const CopyToRoutine CopyToRoutineBinary = {
+ .CopyToEstimateStateSpace = CopyToEstimateStateBinary,
.CopyToStart = CopyToBinaryStart,
.CopyToOutFunc = CopyToBinaryOutFunc,
.CopyToOneRow = CopyToBinaryOneRow,
.CopyToEnd = CopyToBinaryEnd,
};
-/* Return a COPY TO routine for the given options */
-static const CopyToRoutine *
-CopyToGetRoutine(const CopyFormatOptions *opts)
+static Size
+CopyToEstimateStateTextLike(void)
{
- if (opts->csv_mode)
- return &CopyToRoutineCSV;
- else if (opts->binary)
- return &CopyToRoutineBinary;
-
- /* default is text */
- return &CopyToRoutineText;
+ return sizeof(CopyToStateTextLike);
}
/* Implementation of the start callback for text and CSV formats */
static void
-CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc)
+CopyToTextLikeStart(CopyToState ccstate, TupleDesc tupDesc)
{
+ CopyToStateTextLike *cstate = (CopyToStateTextLike *) ccstate;
+
+ /* Use client encoding when ENCODING option is not specified. */
+ if (cstate->base.opts.file_encoding < 0)
+ cstate->file_encoding = pg_get_client_encoding();
+ else
+ cstate->file_encoding = cstate->base.opts.file_encoding;
+
+ /*
+ * Set up encoding conversion info if the file and server encodings differ
+ * (see also pg_server_to_any).
+ */
+ if (cstate->file_encoding == GetDatabaseEncoding() ||
+ cstate->file_encoding == PG_SQL_ASCII)
+ cstate->need_transcoding = false;
+ else
+ cstate->need_transcoding = true;
+
+ /* See Multibyte encoding comment above */
+ cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
+
/*
* For non-binary copy, we need to convert null_print to file encoding,
* because it will be sent directly with CopySendString.
*/
if (cstate->need_transcoding)
- cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
- cstate->opts.null_print_len,
- cstate->file_encoding);
+ cstate->base.opts.null_print_client = pg_server_to_any(cstate->base.opts.null_print,
+ cstate->base.opts.null_print_len,
+ cstate->file_encoding);
/* if a header has been requested send the line */
- if (cstate->opts.header_line == COPY_HEADER_TRUE)
+ if (cstate->base.opts.header_line == COPY_HEADER_TRUE)
{
ListCell *cur;
bool hdr_delim = false;
- foreach(cur, cstate->attnumlist)
+ foreach(cur, cstate->base.attnumlist)
{
int attnum = lfirst_int(cur);
char *colname;
if (hdr_delim)
- CopySendChar(cstate, cstate->opts.delim[0]);
+ CopySendChar(ccstate, cstate->base.opts.delim[0]);
hdr_delim = true;
colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
- if (cstate->opts.csv_mode)
+ if (cstate->base.opts.csv_mode)
CopyAttributeOutCSV(cstate, colname, false);
else
CopyAttributeOutText(cstate, colname);
}
- CopySendTextLikeEndOfRow(cstate);
+ CopySendTextLikeEndOfRow(ccstate);
}
}
@@ -294,10 +276,10 @@ CopyToTextLikeOneRow(CopyToState cstate,
value);
if (is_csv)
- CopyAttributeOutCSV(cstate, string,
+ CopyAttributeOutCSV((CopyToStateTextLike *) cstate, string,
cstate->opts.force_quote_flags[attnum - 1]);
else
- CopyAttributeOutText(cstate, string);
+ CopyAttributeOutText((CopyToStateTextLike *) cstate, string);
}
}
@@ -311,6 +293,13 @@ CopyToTextLikeEnd(CopyToState cstate)
/* Nothing to do here */
}
+static Size
+CopyToEstimateStateBinary(void)
+{
+ /* Binary format doesn't require additional fields */
+ return sizeof(CopyToStateData);
+}
+
/*
* Implementation of the start callback for binary format. Send a header
* for a binary copy.
@@ -406,7 +395,7 @@ SendCopyBegin(CopyToState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->copy_dest = COPY_DEST_FRONTEND;
}
static void
@@ -453,7 +442,7 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
ferror(cstate->copy_file))
@@ -487,11 +476,11 @@ CopySendEndOfRow(CopyToState cstate)
errmsg("could not write to COPY file: %m")));
}
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
- case COPY_CALLBACK:
+ case COPY_DEST_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
break;
}
@@ -512,7 +501,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
{
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
/* Default line termination depends on platform */
#ifndef WIN32
CopySendChar(cstate, '\n');
@@ -520,7 +509,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
CopySendString(cstate, "\r\n");
#endif
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* The FE/BE protocol uses \n as newline for all platforms */
CopySendChar(cstate, '\n');
break;
@@ -614,6 +603,54 @@ EndCopy(CopyToState cstate)
pfree(cstate);
}
+/*
+ * Allocate COPY TO state data based on the format's EsimateStateSpace
+ * callback.
+ */
+static CopyToState
+create_copyto_state(ParseState *pstate, List *options)
+{
+ const CopyToRoutine *routine;
+ CopyToState cstate;
+ Size req_size;
+ bool format_specified = false;
+
+ routine = &CopyToRoutineText; /* default */
+ foreach_node(DefElem, defel, options)
+ {
+ if (strcmp(defel->defname, "format") == 0)
+ {
+ char *fmt = defGetString(defel);
+
+ if (format_specified)
+ errorConflictingDefElem(defel, pstate);
+ format_specified = true;
+ if (strcmp(fmt, "text") == 0)
+ routine = &CopyToRoutineText;
+ else if (strcmp(fmt, "csv") == 0)
+ routine = &CopyToRoutineCSV;
+ else if (strcmp(fmt, "binary") == 0)
+ routine = &CopyToRoutineBinary;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", fmt),
+ parser_errposition(pstate, defel->location)));
+
+ break;
+ }
+ }
+
+ req_size = routine->CopyToEstimateStateSpace();
+ Assert(req_size >= sizeof(CopyToStateData));
+
+ /* Allocate workspace and zero all fields */
+ cstate = (CopyToState) palloc0(req_size);
+ cstate->routine = routine;
+
+ return cstate;
+}
+
/*
* Setup CopyToState to read tuples from a table or a query for COPY TO.
*
@@ -718,9 +755,7 @@ BeginCopyTo(ParseState *pstate,
RelationGetRelationName(rel))));
}
-
- /* Allocate workspace and zero all fields */
- cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateData));
+ cstate = create_copyto_state(pstate, options);
/*
* We allocate everything used by a cstate in a new memory context. This
@@ -735,9 +770,6 @@ BeginCopyTo(ParseState *pstate,
/* Extract options from the statement node tree */
ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options);
- /* Set format routine */
- cstate->routine = CopyToGetRoutine(&cstate->opts);
-
/* Process the source/target relation or query */
if (rel)
{
@@ -918,31 +950,12 @@ BeginCopyTo(ParseState *pstate,
}
}
- /* Use client encoding when ENCODING option is not specified. */
- if (cstate->opts.file_encoding < 0)
- cstate->file_encoding = pg_get_client_encoding();
- else
- cstate->file_encoding = cstate->opts.file_encoding;
-
- /*
- * Set up encoding conversion info if the file and server encodings differ
- * (see also pg_server_to_any).
- */
- if (cstate->file_encoding == GetDatabaseEncoding() ||
- cstate->file_encoding == PG_SQL_ASCII)
- cstate->need_transcoding = false;
- else
- cstate->need_transcoding = true;
-
- /* See Multibyte encoding comment above */
- cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
-
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->copy_dest = COPY_DEST_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->copy_dest = COPY_DEST_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
@@ -1233,16 +1246,16 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
#define DUMPSOFAR() \
do { \
if (ptr > start) \
- CopySendData(cstate, start, ptr - start); \
+ CopySendData((CopyToState) cstate, start, ptr - start); \
} while (0)
static void
-CopyAttributeOutText(CopyToState cstate, const char *string)
+CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string)
{
const char *ptr;
const char *start;
char c;
- char delimc = cstate->opts.delim[0];
+ char delimc = cstate->base.opts.delim[0];
if (cstate->need_transcoding)
ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
@@ -1307,14 +1320,14 @@ CopyAttributeOutText(CopyToState cstate, const char *string)
}
/* if we get here, we need to convert the control char */
DUMPSOFAR();
- CopySendChar(cstate, '\\');
- CopySendChar(cstate, c);
+ CopySendChar((CopyToState) cstate, '\\');
+ CopySendChar((CopyToState) cstate, c);
start = ++ptr; /* do not include char in next run */
}
else if (c == '\\' || c == delimc)
{
DUMPSOFAR();
- CopySendChar(cstate, '\\');
+ CopySendChar((CopyToState) cstate, '\\');
start = ptr++; /* we include char in next run */
}
else if (IS_HIGHBIT_SET(c))
@@ -1367,14 +1380,14 @@ CopyAttributeOutText(CopyToState cstate, const char *string)
}
/* if we get here, we need to convert the control char */
DUMPSOFAR();
- CopySendChar(cstate, '\\');
- CopySendChar(cstate, c);
+ CopySendChar((CopyToState) cstate, '\\');
+ CopySendChar((CopyToState) cstate, c);
start = ++ptr; /* do not include char in next run */
}
else if (c == '\\' || c == delimc)
{
DUMPSOFAR();
- CopySendChar(cstate, '\\');
+ CopySendChar((CopyToState) cstate, '\\');
start = ptr++; /* we include char in next run */
}
else
@@ -1390,19 +1403,19 @@ CopyAttributeOutText(CopyToState cstate, const char *string)
* CSV-style escaping
*/
static void
-CopyAttributeOutCSV(CopyToState cstate, const char *string,
+CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string,
bool use_quote)
{
const char *ptr;
const char *start;
char c;
- char delimc = cstate->opts.delim[0];
- char quotec = cstate->opts.quote[0];
- char escapec = cstate->opts.escape[0];
- bool single_attr = (list_length(cstate->attnumlist) == 1);
+ char delimc = cstate->base.opts.delim[0];
+ char quotec = cstate->base.opts.quote[0];
+ char escapec = cstate->base.opts.escape[0];
+ bool single_attr = (list_length(cstate->base.attnumlist) == 1);
/* force quoting if it matches null_print (before conversion!) */
- if (!use_quote && strcmp(string, cstate->opts.null_print) == 0)
+ if (!use_quote && strcmp(string, cstate->base.opts.null_print) == 0)
use_quote = true;
if (cstate->need_transcoding)
@@ -1445,7 +1458,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
if (use_quote)
{
- CopySendChar(cstate, quotec);
+ CopySendChar((CopyToState) cstate, quotec);
/*
* We adopt the same optimization strategy as in CopyAttributeOutText
@@ -1456,7 +1469,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
if (c == quotec || c == escapec)
{
DUMPSOFAR();
- CopySendChar(cstate, escapec);
+ CopySendChar((CopyToState) cstate, escapec);
start = ptr; /* we include char in next run */
}
if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
@@ -1466,12 +1479,12 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string,
}
DUMPSOFAR();
- CopySendChar(cstate, quotec);
+ CopySendChar((CopyToState) cstate, quotec);
}
else
{
/* If it doesn't need quoting, we can just dump it as-is */
- CopySendString(cstate, ptr);
+ CopySendString((CopyToState) cstate, ptr);
}
}
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 541176e1980..d75a70715a4 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -87,10 +87,12 @@ typedef struct CopyFormatOptions
List *convert_select; /* list of column names (can be NIL) */
} CopyFormatOptions;
-/* These are private in commands/copy[from|to].c */
-typedef struct CopyFromStateData *CopyFromState;
+/* defined in copystate.h */
typedef struct CopyToStateData *CopyToState;
+/* Private in commands/copyfrom.c */
+typedef struct CopyFromStateData *CopyFromState;
+
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
typedef void (*copy_data_dest_cb) (void *data, int len);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 2a2d2f9876b..3b9982d54b8 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -22,6 +22,11 @@
*/
typedef struct CopyToRoutine
{
+ /*
+ * Estimate and return the memory size required to store the state data.
+ */
+ Size (*CopyToEstimateStateSpace) (void);
+
/*
* Set output function information. This callback is called once at the
* beginning of COPY TO.
diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h
new file mode 100644
index 00000000000..7561083a323
--- /dev/null
+++ b/src/include/commands/copystate.h
@@ -0,0 +1,68 @@
+/*-------------------------------------------------------------------------
+ *
+ * copystate.h
+ * Definitions for state data used by COPY TO and COPY FROM.
+ *
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/commands/copystate.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef COPYSTATE_H
+#define COPYSTATE_H
+
+#include "postgres.h"
+#include "commands/copy.h"
+#include "executor/execdesc.h"
+
+/*
+ * Represents the different dest cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopyDest
+{
+ COPY_DEST_FILE, /* to/from file (or a piped program) */
+ COPY_DEST_FRONTEND, /* to/from frontend */
+ COPY_DEST_CALLBACK, /* to/from callback function */
+} CopyDest;
+
+/*
+ * This struct contains the state variables commonly used throughout a
+ * COPY TO operation.
+ */
+typedef struct CopyToStateData
+{
+ /* format-specific routines */
+ const struct CopyToRoutine *routine;
+
+ /* low-level state data */
+ CopyDest copy_dest; /* type of copy source/destination */
+ FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */
+ StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+ List *partitions; /* OID list of partitions to copy data from */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+ MemoryContext rowcontext; /* per-row evaluation context */
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyToStateData;
+
+#endif
--
2.47.3
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-11-17 06:40 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-11-17 06:40 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 14 Nov 2025 12:19:47 -0800,
Masahiko Sawada <[email protected]> wrote:
> This thread has involved extensive discussion, and the patch needs to
> be rebased. I'd like to summarize the current status of this patch and
> our discussions. I've attached updated patches that implement the
> whole ideas of this feature to help provide a clearer overall picture.
Thanks!
I'm not sure whether we should include option parsing
feature to this patch's scope or not but I'm OK with this
approach.
Here are my review comments but they are minor
comments. They don't require design change.
0001:
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cef452584e5..6a0a66507ba 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -155,6 +120,7 @@ static void CopySendInt16(CopyToState cstate, int16 val);
/* text format */
static const CopyToRoutine CopyToRoutineText = {
+ .CopyToEstimateStateSpace = CopyToEstimateStateTextLike,
How about including "Space"
(CopyToEstimateStateSpaceTextLike)?
How about renaming this to
"CopyToTextLikeEstimateStateSpace" because other functions
use "CopyToTextLikeXXX" style.
@@ -171,62 +138,77 @@ static const CopyToRoutine CopyToRoutineCSV = {
...
/* Implementation of the start callback for text and CSV formats */
static void
-CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc)
+CopyToTextLikeStart(CopyToState ccstate, TupleDesc tupDesc)
{
+ CopyToStateTextLike *cstate = (CopyToStateTextLike *) ccstate;
How about always using "cstate" for "CopyToState"? In other
functions in this patch, "CopyToState" is referred "cstate"
or "cctate".
We can use "state" or something for
"CopyToStateTextLike". (0002 uses "state" for
"CopyFromStateBuiltins".)
+ cstate->base.opts.null_print_client = pg_server_to_any(cstate->base.opts.null_print,
+ cstate->base.opts.null_print_len,
+ cstate->file_encoding);
We can use "ccstate->" instead of "cstate->base." in this
function.
@@ -614,6 +603,54 @@ EndCopy(CopyToState cstate)
pfree(cstate);
}
+/*
+ * Allocate COPY TO state data based on the format's EsimateStateSpace
+ * callback.
+ */
+static CopyToState
+create_copyto_state(ParseState *pstate, List *options)
"Esimate" ->
"Estimate"
How about using CamelCase like "CreateCopyToState" for
function name like other functions in this file?
+ Size req_size;
"state_size" may be better.
@@ -1233,16 +1246,16 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
...
static void
-CopyAttributeOutText(CopyToState cstate, const char *string)
+CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string)
{
CopyAttributeOutText(CopyToState cstate, const char *string)
{
CopyToStateTextLike *state = (CopyToStateTextLike *)cstate;
may reduce "cstate->base." and "(CopyToState) cstate" in
this function.
BTW, could you remove a needless space after "*" in
"CopyToStateTextLike * cstate"?
0002:
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 12781963b4f..0c51e5ba5e1 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -129,6 +131,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate);
/* text format */
static const CopyFromRoutine CopyFromRoutineText = {
+ .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace,
How about including "State"
("CopyFromBuiltinsEstimateStateSpace")?
@@ -145,54 +149,129 @@ static const CopyFromRoutine CopyFromRoutineCSV = {
...
+/*
+ * Common routine to initialize CopyFromStateBuiltins data.
+ */
+static void
+initialize_copyfrom_bultins_state(CopyFromStateBuiltins * state, TupleDesc tupDesc)
* How about using CamelCase like other functions in this file?
* How about using the same name as the struct?
InitializeCopyFromStateBuiltins?
@@ -1379,8 +1462,8 @@ CopyFrom(CopyFromState cstate)
/* Add this tuple to the tuple buffer */
CopyMultiInsertInfoStore(&multiInsertInfo,
resultRelInfo, myslot,
- cstate->line_buf.len,
- cstate->cur_lineno);
+ rowinfo.tuplen,
+ rowinfo.lineno);
How about passing "CopyFromRowInfo *" instead of
"rowinfo.tuplen" and "rowinfo.lineno"?
@@ -1512,6 +1595,50 @@ CopyFrom(CopyFromState cstate)
return processed;
}
+static CopyFromState
+create_copyfrom_state(ParseState *pstate, List *options)
How about using CamelCase like other functions in this file?
CreateCopyFromState?
@@ -1138,19 +1156,29 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
...
+ if (rowinfo)
+ {
+ /*
+ * XXX: We used to use line_buf.len but we don't actually use line_buf
+ * in binary format.
+ */
+ rowinfo->lineno = cstate->base.cur_lineno;
+ rowinfo->tuplen = cstate->line_buf.len;
}
How about always setting "0" to "rowinfo->tuplen" instead of
using "cstate->line_buf.len"?
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index d75a70715a4..30a1d2bff6e 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -48,6 +48,12 @@ typedef enum CopyLogVerbosityChoice
...
+typedef struct CopyFromRowInfo
+{
+ uint64 lineno;
+ int tuplen;
+} CopyFromRowInfo;
I don't have a strong opinion nor alternative but I'm not
sure whether this name is suitable or not...
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 3b9982d54b8..c3d2199a0b6 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -99,12 +104,11 @@ typedef struct CopyFromRoutine
* Returns false if there are no more tuples to read.
*/
bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls);
+ Datum *values, bool *nulls, CopyFromRowInfo * rowinfo);
Can we add some docstrings for "rowinfo"?
diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h
index 7561083a323..145dccd0f8f 100644
--- a/src/include/commands/copystate.h
+++ b/src/include/commands/copystate.h
@@ -65,4 +67,99 @@ typedef struct CopyToStateData
...
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_FILE, /* from file (or a piped program) */
+ COPY_FRONTEND, /* from frontend */
+ COPY_CALLBACK, /* from callback function */
+} CopySource;
Can we use "COPY_SOURCE_" prefix instead of "COPY_" prefix
such as "COPY_SOURCE_FILE"?
0003:
diff --git a/src/backend/commands/copy_custom_format.c b/src/backend/commands/copy_custom_format.c
new file mode 100644
index 00000000000..8bef6e779ac
--- /dev/null
+++ b/src/backend/commands/copy_custom_format.c
+/*
+ * Returns true if the given custom format name is registered.
+ */
+bool
+FindCustomCopyFormat(const char *fmt_name)
+{
+ for (int i = 0; i < CopyCustomFormatAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0)
+ return true;
+ }
+
+ return false;
+}
How about using other word than "Find" here? I expect that
"FindXXX()" returns a found value instead of "whether found
or not" as bool.
CustomCopyFormatExists()?
+bool
+GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine)
+{
+ for (int i = 0; i < CopyCustomFormatAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0)
+ {
+ *to_routine = CopyCustomFormatArray[i].to_routine;
+ return true;
+ }
+ }
+
+ return false;
+}
How about returning "const CopyToRoutine *" instead of
"bool"? We can use "FindCustomCopyFormat()" whether the
given format name exists or not.
+bool
+GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine)
Ditto.
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 30a1d2bff6e..82f07b05823 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -120,6 +120,12 @@ extern uint64 CopyFrom(CopyFromState cstate);
extern DestReceiver *CreateCopyDestReceiver(void);
+extern void ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out,
+ bool is_from, List **other_options, ParseState *pstate);
It seems that this change should exist in 0004.
0004:
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b1b3ae141eb..ea31fa911f9 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -1570,3 +1571,32 @@ CreateCopyDestReceiver(void)
return (DestReceiver *) self;
}
+
+static void
+ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate)
+{
+ bool temp_state = false;
+ List *other_options = NIL;
+ CopyFormatOptions *opts;
+
+ if (cstate == NULL)
+ {
+ cstate = create_copyto_state(pstate, options);
+ temp_state = true;
+ }
+
+ opts = &cstate->opts;
+
+ ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate);
+
+ foreach_node(DefElem, option, options)
options ->
other_options
(This change exists in 0005.)
0006:
diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
new file mode 100644
index 00000000000..a63390e875b
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
+static Size
+TestCopyToEsimateSpace(void)
+{
+ return sizeof(TestCopyToState);
+}
+
+static Size
+TestCopyFromEsimateSpace(void)
+{
+ return sizeof(TestCopyFromState);
+}
EsimateSpace ->
EstimateStateSpace
+ if (strcmp(option->defname, "common_int") == 0)
+ {
+ int val = defGetInt32(option);
+
+ opt->common_int = val;
+
+ return true;
+ }
+ else if (strcmp(option->defname, "common_bool") == 0)
+ {
+ bool val = defGetBoolean(option);
+
+ opt->common_bool = val;
+
+ return true;
We may not need to use local variables:
opt->common_int = defGetInt32(option);
opt->common_bool = defGetBoolean(option);
> One potential improvement would be adding support for random file
> access in COPY FROM operations. For example, with parquet files, it
> would be much more efficient to read the footer section first since it
> contains metadata, allowing selective reading of necessary file
> sections. The current sequential read API (CopyFromGetData()) requires
> reading all data to access the metadata.
This is outside the scope of this patch but I've created
a custom COPY format implementation for Apache Parquet:
https://github.com/kou/pg-copy-parquet
We need to start parsing from footer as mentioned above. So
the implementation reads all data before it starts parsing:
https://github.com/kou/pg-copy-parquet/blob/7da367ea81d8964f5045fe0b1514a798d4ecbbc7/copy_parquet.cc...
If we have random access API, we don't need to read all
data. It improves performance.
Anyway, this is outside the scope of this patch. We can
discuss this in a separated thread after we merge this
patch.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-11-17 17:04 ` Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Tomas Vondra @ 2025-11-17 17:04 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On 11/14/25 21:19, Masahiko Sawada wrote:
> After offline discussions with Sutou-san, we believe the current APIs
> work well, particularly for text-based formats, though we still need
> to verify there are no performance regressions.
I got pinged about this patch off-list. I won't have capacity to do a
proper review, anytime soon, but I got a bit of time to do a simple
benchmark (which seems useful as that was one of the concerns in this
thread, it seems).
Attached is a script that does COPY TO/FROM with the built-in formats,
on table with 1, 10 and 100 integer columns. The data sets are between
10 and 1M rows. The table is UNLOGGED, to eliminate WAL overhead.
The attached PDF summarizes results from my ryzen machine, for master
and patched build. The final columns are comparison (i.e. copy/master)
of the timings. Values >100% are regressions (marked as red).
It seems quite "red", but it's not particularly conclusive. The
differences are mostly within 5%, and that could be caused e.g. by
changes to binary layout. And some of the cases got faster too.
It might be interesting to get results from other machines. The script
may need some adjustments, but it should be too difficult.
The other thing Andres was concerned about is "the amount of COPY
performance improvements it forecloses". I have no opinion on that, as
it depends on what improvements Andres envisioned.
regards
--
Tomas Vondra
Attachments:
[application/x-shellscript] copy.sh (1.9K, ../../[email protected]/2-copy.sh)
download
[application/pdf] copy.pdf (46.4K, ../../[email protected]/3-copy.pdf)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
@ 2025-12-02 02:39 ` Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2025-12-02 02:39 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Nov 2025 18:04:46 +0100,
Tomas Vondra <[email protected]> wrote:
> I got pinged about this patch off-list. I won't have capacity to do a
> proper review, anytime soon, but I got a bit of time to do a simple
> benchmark (which seems useful as that was one of the concerns in this
> thread, it seems).
Thanks!!!
I also do the same condition benchmark on my Mac mini:
Machine:
* Apple M1 (8 core)
* Memory: 16GB
* macOS Sequoia (15.6)
Parameters:
* N integer columns: 1 10 100
* N rows: 10 100 1000 10000 10000 100000 1000000
* Formats: text csv binary
* Operations: FROM TO
* Patches: 0001 0002 0003 0004 0005 0006
I ran 5 times for each parameter set and choose the median
elapsed time.
I used
https://gitlab.com/ktou/pg-bench/-/blob/main/copy-format-extendable/run.sh
. I attach it.
I measured 6 times. See the attached mac-mini-result-${N}.{csv,pdf}.
A PDF has 3 columns. They are text, csv and binary formats
from left to right.
1st row uses COPY FROM and 0001 patch.
2nd row uses COPY TO and 0001 patch.
3rd row uses COPY FROM and 0002 patch.
4th row uses COPY TO and 0002 patch.
...
Each heatmap visualizes (${elapsed_time_patch} /
${elapsed_time_master}) * 100. 100 > (red) means slower and
100 < (blue) means faster.
It seems that they don't show any reproducible trends.
For example, the binary cases in mac-mini-result-2.pdf show
that patched cases are always slower but the binary cases in
mac-mini-result-{1,6}.pdf show that most patched cases are
faster. The binary cases in mac-mini-result-{1,5,6}.pdf show
that 0006 patch is slower than 0005 patch but the binary
cases in mac-mini-result-{3,4}.pdf don't show it.
Another example, the text cases in mac-mini-result-1.pdf
show that patched cases are always slower but the text cases
in mac-mini-result-1.pdf show that most patched cases are
faster.
I hope that these numbers help to proceed this proposal.
Thanks,
--
kou
#!/bin/bash
# -*- sh-indentation: 2; sh-basic-offset: 2-*-
# This is based on codes from Michael Paquier and Tomas Vondra:
# * https://www.postgresql.org/message-id/flat/Zbr6piWuVHDtFFOl%40paquier.xyz#dbbec4d5c54ef2317be01a54ab...
# * https://www.postgresql.org/message-id/flat/26541788-8853-4d93-86cd-5f701b13ae51%40enterprisedb.com
DIR=${1:-$(pwd)}
psql postgres > /dev/null 2>&1 <<EOF
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
EOF
psql test > /dev/null 2>&1 <<EOF
CREATE EXTENSION blackhole_am;
CREATE OR REPLACE FUNCTION create_table_cols(tabname text, num_cols int)
RETURNS VOID AS
\$func\$
DECLARE
query text;
BEGIN
query := 'CREATE UNLOGGED TABLE ' || tabname || ' (';
FOR i IN 1..num_cols LOOP
query := query || 'a_' || i::text || ' int2 default 1';
IF i != num_cols THEN
query := query || ', ';
END IF;
END LOOP;
query := query || ')';
EXECUTE format(query);
END
\$func\$ LANGUAGE plpgsql;
EOF
for format in text csv binary; do
for n_columns in 1 10 100; do
for n_rows in 10 100 1000 10000 10000 100000 1000000; do
psql test > /dev/null 2>&1 <<EOF
DROP TABLE IF EXISTS to_table;
DROP TABLE IF EXISTS from_table;
SELECT create_table_cols ('to_table', ${n_columns});
SELECT create_table_cols ('from_table', ${n_columns});
INSERT INTO to_table SELECT 1 FROM generate_series(1, ${n_rows});
COPY to_table TO '$DIR/test.data' WITH (FORMAT ${format});
ALTER TABLE from_table SET ACCESS METHOD blackhole_am;
EOF
# run COPY TO 5x
for r in $(seq 1 5); do
s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())")
psql test -c "COPY to_table TO '/dev/null' WITH (FORMAT ${format})" > /dev/null 2>&1
d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)")
echo "TO,${format},${n_columns},${n_rows},${r},${d}"
done
# run COPY FROM 5x
for r in $(seq 1 5); do
s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())")
psql test -c "COPY from_table FROM '$DIR/test.data' WITH (FORMAT ${format})" > /dev/null 2>&1
d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)")
echo "FROM,${format},${n_columns},${n_rows},${r},${d}"
done
done
done
done
Attachments:
[text/plain] run.sh (2.2K, ../../[email protected]/2-run.sh)
download | inline:
#!/bin/bash
# -*- sh-indentation: 2; sh-basic-offset: 2-*-
# This is based on codes from Michael Paquier and Tomas Vondra:
# * https://www.postgresql.org/message-id/flat/Zbr6piWuVHDtFFOl%40paquier.xyz#dbbec4d5c54ef2317be01a54abaf495c
# * https://www.postgresql.org/message-id/flat/26541788-8853-4d93-86cd-5f701b13ae51%40enterprisedb.com
DIR=${1:-$(pwd)}
psql postgres > /dev/null 2>&1 <<EOF
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
EOF
psql test > /dev/null 2>&1 <<EOF
CREATE EXTENSION blackhole_am;
CREATE OR REPLACE FUNCTION create_table_cols(tabname text, num_cols int)
RETURNS VOID AS
\$func\$
DECLARE
query text;
BEGIN
query := 'CREATE UNLOGGED TABLE ' || tabname || ' (';
FOR i IN 1..num_cols LOOP
query := query || 'a_' || i::text || ' int2 default 1';
IF i != num_cols THEN
query := query || ', ';
END IF;
END LOOP;
query := query || ')';
EXECUTE format(query);
END
\$func\$ LANGUAGE plpgsql;
EOF
for format in text csv binary; do
for n_columns in 1 10 100; do
for n_rows in 10 100 1000 10000 10000 100000 1000000; do
psql test > /dev/null 2>&1 <<EOF
DROP TABLE IF EXISTS to_table;
DROP TABLE IF EXISTS from_table;
SELECT create_table_cols ('to_table', ${n_columns});
SELECT create_table_cols ('from_table', ${n_columns});
INSERT INTO to_table SELECT 1 FROM generate_series(1, ${n_rows});
COPY to_table TO '$DIR/test.data' WITH (FORMAT ${format});
ALTER TABLE from_table SET ACCESS METHOD blackhole_am;
EOF
# run COPY TO 5x
for r in $(seq 1 5); do
s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())")
psql test -c "COPY to_table TO '/dev/null' WITH (FORMAT ${format})" > /dev/null 2>&1
d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)")
echo "TO,${format},${n_columns},${n_rows},${r},${d}"
done
# run COPY FROM 5x
for r in $(seq 1 5); do
s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())")
psql test -c "COPY from_table FROM '$DIR/test.data' WITH (FORMAT ${format})" > /dev/null 2>&1
d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)")
echo "FROM,${format},${n_columns},${n_rows},${r},${d}"
done
done
done
done
[text/csv] mac-mini-result-1.csv (67.2K, ../../[email protected]/3-mac-mini-result-1.csv)
download
[application/pdf] mac-mini-result-1.pdf (823.5K, ../../[email protected]/4-mac-mini-result-1.pdf)
download
[text/csv] mac-mini-result-2.csv (66.6K, ../../[email protected]/5-mac-mini-result-2.csv)
download
[application/pdf] mac-mini-result-2.pdf (814.9K, ../../[email protected]/6-mac-mini-result-2.pdf)
download
[text/csv] mac-mini-result-3.csv (67.1K, ../../[email protected]/7-mac-mini-result-3.csv)
download
[application/pdf] mac-mini-result-3.pdf (823.8K, ../../[email protected]/8-mac-mini-result-3.pdf)
download
[text/csv] mac-mini-result-4.csv (66.7K, ../../[email protected]/9-mac-mini-result-4.csv)
download
[application/pdf] mac-mini-result-4.pdf (824.2K, ../../[email protected]/10-mac-mini-result-4.pdf)
download
[text/csv] mac-mini-result-5.csv (67.2K, ../../[email protected]/11-mac-mini-result-5.csv)
download
[application/pdf] mac-mini-result-5.pdf (823.4K, ../../[email protected]/12-mac-mini-result-5.pdf)
download
[text/csv] mac-mini-result-6.csv (67.2K, ../../[email protected]/13-mac-mini-result-6.csv)
download
[application/pdf] mac-mini-result-6.pdf (823.9K, ../../[email protected]/14-mac-mini-result-6.pdf)
download
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-12-18 23:43 ` Masahiko Sawada <[email protected]>
2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-12-18 23:43 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Dec 1, 2025 at 6:41 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Nov 2025 18:04:46 +0100,
> Tomas Vondra <[email protected]> wrote:
>
> > I got pinged about this patch off-list. I won't have capacity to do a
> > proper review, anytime soon, but I got a bit of time to do a simple
> > benchmark (which seems useful as that was one of the concerns in this
> > thread, it seems).
>
> Thanks!!!
>
> I also do the same condition benchmark on my Mac mini:
>
> Machine:
>
> * Apple M1 (8 core)
> * Memory: 16GB
> * macOS Sequoia (15.6)
>
> Parameters:
>
> * N integer columns: 1 10 100
> * N rows: 10 100 1000 10000 10000 100000 1000000
> * Formats: text csv binary
> * Operations: FROM TO
> * Patches: 0001 0002 0003 0004 0005 0006
>
> I ran 5 times for each parameter set and choose the median
> elapsed time.
>
> I used
> https://gitlab.com/ktou/pg-bench/-/blob/main/copy-format-extendable/run.sh
> . I attach it.
>
> I measured 6 times. See the attached mac-mini-result-${N}.{csv,pdf}.
>
> A PDF has 3 columns. They are text, csv and binary formats
> from left to right.
>
> 1st row uses COPY FROM and 0001 patch.
> 2nd row uses COPY TO and 0001 patch.
>
> 3rd row uses COPY FROM and 0002 patch.
> 4th row uses COPY TO and 0002 patch.
>
> ...
>
> Each heatmap visualizes (${elapsed_time_patch} /
> ${elapsed_time_master}) * 100. 100 > (red) means slower and
> 100 < (blue) means faster.
>
> It seems that they don't show any reproducible trends.
>
> For example, the binary cases in mac-mini-result-2.pdf show
> that patched cases are always slower but the binary cases in
> mac-mini-result-{1,6}.pdf show that most patched cases are
> faster. The binary cases in mac-mini-result-{1,5,6}.pdf show
> that 0006 patch is slower than 0005 patch but the binary
> cases in mac-mini-result-{3,4}.pdf don't show it.
>
> Another example, the text cases in mac-mini-result-1.pdf
> show that patched cases are always slower but the text cases
> in mac-mini-result-1.pdf show that most patched cases are
> faster.
>
> I hope that these numbers help to proceed this proposal.
Thank you for sharing the performance test results! I'll run the same
benchmark tests on my environment.
Looking at these results, it seems that 0001-from-binary cases and
0006-to-binary cases are slower throughout the six results?
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2026-06-23 01:06 ` Masahiko Sawada <[email protected]>
2026-06-23 05:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2026-06-23 01:06 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
On Thu, Mar 26, 2026 at 6:36 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoCLxUhQ0uBjDKXvCEtJBCfF13Ru_7u-Qrrsu+0PPUqcPQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 18 Dec 2025 15:43:07 -0800,
> Masahiko Sawada <[email protected]> wrote:
>
> > Looking at these results, it seems that 0001-from-binary cases and
> > 0006-to-binary cases are slower throughout the six results?
>
> Good point. I didn't notice them. But I feel that it's not
> related to the patch set. Because 0001 doesn't change COPY
> FROM related code. 0001 just changes COPY TO related
> code. And 0006 just adds tests. 0006 doesn't change
> implementations.
>
>
> BTW, how to proceed this proposal? It seems that we can't
> proceed this proposal without PostgreSQL committers'
> attentions but it seems that it's difficult.
Sorry for going quiet on this for a while -- I haven't had time to
work on it until now.
After more thought, I'd like to keep the custom-format changes to the
bare minimum and not disturb the existing built-in format processing.
In particular, I've dropped the earlier rework that split
CopyToStateData / CopyFromStateData to hide built-in-specific fields
from extensions. That was my own idea, but I no longer think it pays
off: the fields it hid (raw_buf, line_buf, the input buffers, etc.)
are only ever used by the built-in text/CSV/binary parsers, and a
custom format never touches them -- so visible or not, nothing depends
on them, while splitting the struct is invasive to the existing format
processing. Touching the Copy state structs is fine in itself; it's
the hiding that wasn't worth the cost.
Instead, each state struct just gets one opaque pointer for a custom
format to keep its own state, and the existing code paths are left
alone.
Updated patches attached:
- 0001 moves CopyFromStateData and CopyToStateData to a new
copy_state.h, so extensions can implement their routines without
including the *_internal.h headers. It also drops file_fdw.c's
dependency on copyfrom_internal.h.
- 0002 introduces the registration API and the opaque per-format
pointer in both structs.
- 0003 adds a callback to validate the COPY options as a whole, called
after all options are processed.
- 0004 adds the regression tests.
I'd like to proceed in this direction barring objections. Feedback is
very welcome.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-patch] v2-0002-Allow-extensions-to-register-custom-format-to-COP.patch (19.1K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/2-v2-0002-Allow-extensions-to-register-custom-format-to-COP.patch)
download | inline diff:
From 1684e2394e5557f94c17e39b94351a76e601e00d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 09:21:51 -0700
Subject: [PATCH v2 2/4] Allow extensions to register custom format to COPY TO
and COPY FROM.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
---
src/backend/commands/Makefile | 1 +
src/backend/commands/copy.c | 97 ++++++++++++++++++++--
src/backend/commands/copyapi.c | 131 ++++++++++++++++++++++++++++++
src/backend/commands/copyfrom.c | 4 +-
src/backend/commands/copyto.c | 4 +-
src/backend/commands/meson.build | 1 +
src/include/commands/copy.h | 19 +++++
src/include/commands/copy_state.h | 6 ++
src/include/commands/copyapi.h | 37 +++++++++
src/tools/pgindent/typedefs.list | 1 +
10 files changed, 290 insertions(+), 11 deletions(-)
create mode 100644 src/backend/commands/copyapi.c
diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile
index 5b9d084977e..17b7aa08b55 100644
--- a/src/backend/commands/Makefile
+++ b/src/backend/commands/Makefile
@@ -23,6 +23,7 @@ OBJS = \
constraint.o \
conversioncmds.o \
copy.o \
+ copyapi.o \
copyfrom.o \
copyfromparse.o \
copyto.o \
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 003b70852bb..2fdba026ee0 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -23,6 +23,7 @@
#include "access/xact.h"
#include "catalog/pg_authid.h"
#include "commands/copy.h"
+#include "commands/copyapi.h"
#include "commands/defrem.h"
#include "executor/executor.h"
#include "mb/pg_wchar.h"
@@ -592,6 +593,14 @@ ProcessCopyOptions(ParseState *pstate,
bool force_array_specified = false;
ListCell *option;
+ /*
+ * Options not recognized by core are collected here and, once the format
+ * is known, either handed to a custom format's option parser or rejected.
+ */
+ List *deferred_options = NIL;
+ ProcessOneOptionFn custom_process_option_fn = NULL;
+ char *custom_format_name = NULL;
+
/* Support external use for option sanity checking */
if (opts_out == NULL)
opts_out = palloc0_object(CopyFormatOptions);
@@ -620,6 +629,13 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->format = COPY_FORMAT_BINARY;
else if (strcmp(fmt, "json") == 0)
opts_out->format = COPY_FORMAT_JSON;
+ else if (GetCopyCustomFormatRoutines(fmt, &opts_out->to_routine,
+ &opts_out->from_routine,
+ &custom_process_option_fn))
+ {
+ opts_out->format = COPY_FORMAT_CUSTOM;
+ custom_format_name = fmt;
+ }
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -775,11 +791,54 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->reject_limit = defGetCopyRejectLimitOption(defel);
}
else
+ {
+ /*
+ * Not a core option. Defer the check to after the loop as it may
+ * belong to a custom format whose "format" option has not been
+ * seen yet.
+ */
+ deferred_options = lappend(deferred_options, defel);
+ }
+ }
+
+ /*
+ * Now that the format and every option have been seen, resolve the
+ * deferred options.
+ */
+ if (deferred_options != NIL)
+ {
+ /*
+ * For a custom format, they belong to the handler; for any built-in
+ * (including the default) an unrecognized option is an error,
+ * preserving the historical behavior relied on by external callers
+ * such as file_fdw.
+ */
+ if (opts_out->format != COPY_FORMAT_CUSTOM || custom_process_option_fn == NULL)
+ {
+ DefElem *defel = linitial_node(DefElem, deferred_options);
+
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("option \"%s\" not recognized",
defel->defname),
parser_errposition(pstate, defel->location)));
+ }
+
+ /*
+ * Hand each option core did not recognize to the format's per-option
+ * callback. Anything the format does not claim (or any option at all
+ * if it has no callback) is an error, so an unrecognized option
+ * always fails here.
+ */
+ foreach_node(DefElem, opt, deferred_options)
+ {
+ if (!custom_process_option_fn(opts_out, is_from, opt))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("COPY format \"%s\" does not accept option \"%s\"",
+ custom_format_name, opt->defname),
+ parser_errposition(pstate, opt->location)));
+ }
}
/*
@@ -869,7 +928,7 @@ ProcessCopyOptions(ParseState *pstate,
* future-proofing. Likewise we disallow all digits though only octal
* digits are actually dangerous.
*/
- if (opts_out->format != COPY_FORMAT_CSV &&
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
opts_out->delim[0]) != NULL)
ereport(ERROR,
@@ -888,7 +947,8 @@ ProcessCopyOptions(ParseState *pstate,
: errmsg("cannot specify %s in JSON mode", "HEADER"));
/* Check quote */
- if (opts_out->format != COPY_FORMAT_CSV && opts_out->quote != NULL)
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ opts_out->quote != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -905,7 +965,8 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("COPY delimiter and quote must be different")));
/* Check escape */
- if (opts_out->format != COPY_FORMAT_CSV && opts_out->escape != NULL)
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ opts_out->escape != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -917,7 +978,8 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("COPY escape must be a single one-byte character")));
/* Check force_quote */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_quote || opts_out->force_quote_all))
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_quote || opts_out->force_quote_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -931,8 +993,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY FROM")));
/* Check force_notnull */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_notnull != NIL ||
- opts_out->force_notnull_all))
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_notnull != NIL || opts_out->force_notnull_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -947,8 +1009,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY TO")));
/* Check force_null */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_null != NIL ||
- opts_out->force_null_all))
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_null != NIL || opts_out->force_null_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -995,7 +1057,8 @@ ProcessCopyOptions(ParseState *pstate,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY %s is not supported for %s", "FORMAT JSON", "COPY FROM"));
- if (opts_out->format != COPY_FORMAT_JSON && opts_out->force_array)
+ if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_JSON &&
+ opts_out->force_array)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY %s can only be used with JSON mode", "FORCE_ARRAY"));
@@ -1048,6 +1111,22 @@ ProcessCopyOptions(ParseState *pstate,
* ON_ERROR, third is the value of the COPY option, e.g. IGNORE */
errmsg("COPY %s requires %s to be set to %s",
"REJECT_LIMIT", "ON_ERROR", "IGNORE")));
+
+ /* Check custom format routines */
+ if (opts_out->format == COPY_FORMAT_CUSTOM)
+ {
+ if (is_from && opts_out->from_routine == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY format \"%s\" cannot be used with COPY FROM",
+ custom_format_name)));
+
+ if (!is_from && opts_out->to_routine == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY format \"%s\" cannot be used with COPY TO",
+ custom_format_name)));
+ }
}
/*
diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c
new file mode 100644
index 00000000000..168efbcf30b
--- /dev/null
+++ b/src/backend/commands/copyapi.c
@@ -0,0 +1,131 @@
+/*-------------------------------------------------------------------------
+ *
+ * copyapi.c
+ * Registry for pluggable COPY TO/FROM format handlers.
+ *
+ * The built-in formats (text, csv, binary, json) are dispatched directly by
+ * the COPY engine. Extensions can provide additional formats by registering
+ * a CopyToRoutine and/or CopyFromRoutine under a name from their _PG_init();
+ * ProcessCopyOptions() then resolves "COPY ... (FORMAT 'name')" against this
+ * registry.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/commands/copyapi.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "commands/copyapi.h"
+#include "utils/memutils.h"
+
+typedef struct CopyCustomFormatEntry
+{
+ const char *name; /* constant string; never freed (see below) */
+ const CopyToRoutine *to_routine;
+ const CopyFromRoutine *from_routine;
+ ProcessOneOptionFn option_fn;
+} CopyCustomFormatEntry;
+
+static CopyCustomFormatEntry *CopyCustomFormatArray = NULL;
+static int CopyCustomFormatsAssigned = 0;
+static int CopyCustomFormatsAllocated = 0;
+
+/* Is 'name' one of the built-in format keywords? */
+static bool
+is_builtin_copy_format(const char *name)
+{
+ return (strcmp(name, "text") == 0 ||
+ strcmp(name, "csv") == 0 ||
+ strcmp(name, "binary") == 0 ||
+ strcmp(name, "json") == 0);
+}
+
+/*
+ * Register a custom COPY format. Intended to be called from an extension's
+ * _PG_init(). Either routine may be NULL if the format does not support that
+ * direction (but not both).
+ *
+ * 'option_fn' may also be NULL if the format takes no format-specific options.
+ *
+ * 'name' is assumed to be a constant string or allocated in storage that will
+ * never be freed; it is stored by reference.
+ */
+void
+RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
+ const CopyFromRoutine *from, ProcessOneOptionFn option_fn)
+{
+ Assert(name != NULL && name[0] != '\0');
+
+ /* Must support at least one direction */
+ Assert(to != NULL || from != NULL);
+
+ Assert(to == NULL ||
+ (to->CopyToStart != NULL && to->CopyToOneRow != NULL &&
+ to->CopyToEnd != NULL));
+ Assert(from == NULL ||
+ (from->CopyFromStart != NULL && from->CopyFromOneRow != NULL &&
+ from->CopyFromEnd != NULL));
+
+ /* Check if it's already used by built-in format names */
+ if (is_builtin_copy_format(name))
+ elog(ERROR, "COPY format \"%s\" is a built-in format name", name);
+
+ /* Reject a duplicate registration. */
+ for (int i = 0; i < CopyCustomFormatsAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].name, name) == 0)
+ elog(ERROR, "COPY format \"%s\" is already registered", name);
+ }
+
+ /* Create the array on first use; it must outlive the current context. */
+ if (CopyCustomFormatArray == NULL)
+ {
+ CopyCustomFormatsAllocated = 16;
+ CopyCustomFormatArray = (CopyCustomFormatEntry *)
+ MemoryContextAlloc(TopMemoryContext,
+ CopyCustomFormatsAllocated * sizeof(CopyCustomFormatEntry));
+ }
+
+ /* Expand if full. */
+ if (CopyCustomFormatsAssigned >= CopyCustomFormatsAllocated)
+ {
+ CopyCustomFormatsAllocated *= 2;
+ CopyCustomFormatArray = (CopyCustomFormatEntry *)
+ repalloc_array(CopyCustomFormatArray, CopyCustomFormatEntry, CopyCustomFormatsAllocated);
+ }
+
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].name = name;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn;
+ CopyCustomFormatsAssigned++;
+}
+
+/*
+ * Look up a previously registered custom format. Returns false if 'name' is
+ * not registered. Out-parameters may be NULL if not wanted.
+ */
+bool
+GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
+ const CopyFromRoutine **from, ProcessOneOptionFn * option_fn)
+{
+ for (int i = 0; i < CopyCustomFormatsAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].name, name) == 0)
+ {
+ if (to)
+ *to = CopyCustomFormatArray[i].to_routine;
+ if (from)
+ *from = CopyCustomFormatArray[i].from_routine;
+ if (option_fn)
+ *option_fn = CopyCustomFormatArray[i].option_fn;
+
+ return true;
+ }
+ }
+ return false;
+}
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 2c57b32f4de..69ec94c9ec1 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -158,7 +158,9 @@ static const CopyFromRoutine CopyFromRoutineBinary = {
static const CopyFromRoutine *
CopyFromGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->format == COPY_FORMAT_CSV)
+ if (opts->format == COPY_FORMAT_CUSTOM)
+ return opts->from_routine;
+ else if (opts->format == COPY_FORMAT_CSV)
return &CopyFromRoutineCSV;
else if (opts->format == COPY_FORMAT_BINARY)
return &CopyFromRoutineBinary;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index ef2038c9a5d..f897f23737f 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -130,7 +130,9 @@ static const CopyToRoutine CopyToRoutineBinary = {
static const CopyToRoutine *
CopyToGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->format == COPY_FORMAT_CSV)
+ if (opts->format == COPY_FORMAT_CUSTOM)
+ return opts->to_routine;
+ else if (opts->format == COPY_FORMAT_CSV)
return &CopyToRoutineCSV;
else if (opts->format == COPY_FORMAT_BINARY)
return &CopyToRoutineBinary;
diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build
index 9f258d566eb..d98273da67e 100644
--- a/src/backend/commands/meson.build
+++ b/src/backend/commands/meson.build
@@ -11,6 +11,7 @@ backend_sources += files(
'constraint.c',
'conversioncmds.c',
'copy.c',
+ 'copyapi.c',
'copyfrom.c',
'copyfromparse.c',
'copyto.c',
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 5e710efff5b..9c40ca4ba09 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -58,7 +58,16 @@ typedef enum CopyFormat
COPY_FORMAT_BINARY,
COPY_FORMAT_CSV,
COPY_FORMAT_JSON,
+ COPY_FORMAT_CUSTOM, /* format provided by an extension */
} CopyFormat;
+#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM)
+
+/*
+ * Full definitions live in commands/copyapi.h, which includes this header;
+ * CopyFormatOptions only needs to hold pointers to the resolved routines.
+ */
+struct CopyToRoutine;
+struct CopyFromRoutine;
/*
* A struct to hold COPY options, in a parsed form. All of these are related
@@ -97,6 +106,16 @@ typedef struct CopyFormatOptions
CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */
int64 reject_limit; /* maximum tolerable number of errors */
List *convert_select; /* list of column names (can be NIL) */
+
+ /*
+ * Resolved handler for a custom format. The directoin not in use may be
+ * NULL. For built-in formats these are unused.
+ */
+ const struct CopyToRoutine *to_routine;
+ const struct CopyFromRoutine *from_routine;
+
+ /* Custom format private option data */
+ void *format_private_opts;
} CopyFormatOptions;
/* These are defined in copy_state.h */
diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h
index 52cbf5067eb..6c5defbf4ee 100644
--- a/src/include/commands/copy_state.h
+++ b/src/include/commands/copy_state.h
@@ -178,6 +178,9 @@ typedef struct CopyFromStateData
#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* Custom format private data to store the state */
+ void *format_private;
} CopyFromStateData;
/*
@@ -248,6 +251,9 @@ typedef struct CopyToStateData
FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* Custom format private data to store the state */
+ void *format_private;
} CopyToStateData;
#endif /* COPY_STATE_H */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 398e7a78bb3..8eb5fe9c7dc 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,6 +14,7 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "commands/copy_state.h"
#include "commands/copy.h"
/*
@@ -102,4 +103,40 @@ typedef struct CopyFromRoutine
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
+/*
+ * Optional callback to process one format-specific COPY option. Invoked
+ * from ProcessCopyOptions() once per option that core did not recognize, after
+ * every core option has been parsed (so 'opts' is fully populated).
+ *
+ * Returns true if the option belongs to the format and is valid. Returns false
+ * if the option is not one the format recognizes, in which case core raises the
+ * "not accepted" error; thus an unrecognized option always errors, whether or
+ * not the format supplies this callback. For a recognized option with an invalid
+ * value, the callback should ereport() itself.
+ *
+ * 'pstate' may be NULL (e.g. when options are checked outside a real COPY, as
+ * file_fdw does); parser_errposition(pstate, ...) tolerates NULL.
+ */
+typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from,
+ DefElem *option);
+
+/*
+ * Register a COPY format under 'name', mapping it to its TO and/or FROM
+ * routines and optional option/validation callbacks. Intended to be called
+ * from an extension's _PG_init(). Either routine may be NULL if the format
+ * does not support that direction (but not both). Errors if 'name' collides
+ * with a built-in format or one already registered.
+ */
+extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
+ const CopyFromRoutine *from,
+ ProcessOneOptionFn option_fn);
+
+/*
+ * Look up a previously registered custom format. Returns false if 'name' is
+ * not registered. Out-parameters may be NULL if not wanted.
+ */
+extern bool GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
+ const CopyFromRoutine **from,
+ ProcessOneOptionFn * option_fn);
+
#endif /* COPYAPI_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 1969d467c1d..5263710e451 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -540,6 +540,7 @@ ConvProcInfo
ConversionLocation
ConvertRowtypeExpr
CookedConstraint
+CopyCustomFormatEntry
CopyDest
CopyFormat
CopyFormatOptions
--
2.54.0
[text/x-patch] v2-0004-Add-test-module-for-COPY-custom-format.patch (17.3K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/3-v2-0004-Add-test-module-for-COPY-custom-format.patch)
download | inline diff:
From d31d8cef68d10cb1817446af9a1e492ce88808e9 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 13:14:31 -0700
Subject: [PATCH v2 4/4] Add test module for COPY custom format.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../modules/test_copy_custom_format/Makefile | 20 +++
.../expected/test_copy_custom_format.out | 105 +++++++++++
.../test_copy_custom_format/meson.build | 32 ++++
.../sql/test_copy_custom_format.sql | 32 ++++
.../test_copy_custom_format.c | 169 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
8 files changed, 361 insertions(+)
create mode 100644 src/test/modules/test_copy_custom_format/Makefile
create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
create mode 100644 src/test/modules/test_copy_custom_format/meson.build
create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 0a74ab5c86f..6dcb66174f5 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -23,6 +23,7 @@ SUBDIRS = \
test_cloexec \
test_checksums \
test_copy_callbacks \
+ test_copy_custom_format \
test_custom_rmgrs \
test_custom_stats \
test_custom_types \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 4bca42bb370..adfa413fe58 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -23,6 +23,7 @@ subdir('test_bloomfilter')
subdir('test_cloexec')
subdir('test_checksums')
subdir('test_copy_callbacks')
+subdir('test_copy_custom_format')
subdir('test_cplusplusext')
subdir('test_custom_rmgrs')
subdir('test_custom_stats')
diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile
new file mode 100644
index 00000000000..68a2a04ff09
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/Makefile
@@ -0,0 +1,20 @@
+# src/test/modules/test_copy_custom_format/Makefile
+
+MODULE_big = test_copy_custom_format
+OBJS = \
+ $(WIN32RES) \
+ test_copy_custom_format.o
+PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT"
+
+REGRESS = test_copy_custom_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_copy_custom_format
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
new file mode 100644
index 00000000000..817ca3fa60f
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
@@ -0,0 +1,105 @@
+LOAD 'test_copy_custom_format';
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789);
+COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 2
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToStart: the number of attributes of table: 1, the number of attributes to output: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized
+ERROR: COPY format "nonexistent" not recognized
+LINE 1: COPY copy_data TO stdout WITH (format 'nonexistent');
+ ^
+COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting
+ERROR: conflicting or redundant options
+LINE 1: COPY copy_data TO stdout WITH (format 'text', format 'csv');
+ ^
+COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR
+ERROR: COPY format "test_format" does not accept option "bogus"
+LINE 1: ...PY copy_data TO stdout WITH (format 'test_format', bogus 1);
+ ^
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+ERROR: relation has 3 columns, exceeds max_attributes 2
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive
+ERROR: "max_attributes" must be a positive integer
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR
+ERROR: "max_attributes" must be a positive integer
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required
+ERROR: max_attributes requires an integer value
+COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate)
+ERROR: FREEZE cannot be used with "disallow_freeze" option
+COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+-- The built-in options are handled in the same way of built-in formats.
+COPY copy_data TO stdout WITH (format 'test_format', delimiter ',');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', quote '"');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO
+ERROR: COPY FREEZE cannot be used with COPY TO
diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build
new file mode 100644
index 00000000000..a231ed57649
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/meson.build
@@ -0,0 +1,32 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+test_copy_custom_format_sources = files(
+'test_copy_custom_format.c',
+)
+
+if host_system == 'windows'
+ test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_copy_custom_format',
+ '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',])
+endif
+
+test_copy_custom_format = shared_module('test_copy_custom_format',
+ test_copy_custom_format_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_copy_custom_format
+
+tests += {
+ 'name': 'test_copy_custom_format',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_copy_custom_format',
+ ],
+ # Disabled because these tests require
+ # "shared_preload_libraries=test_custom_copy_format", which typical
+ # runningcheck users do not have (e.g. buildfarm clients).
+ 'runningcheck': false,
+ },
+}
diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
new file mode 100644
index 00000000000..59f58fa55a2
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
@@ -0,0 +1,32 @@
+LOAD 'test_copy_custom_format';
+
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789);
+
+COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End
+COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End
+\.
+
+COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2
+COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1
+
+COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized
+COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting
+
+COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR
+
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required
+
+COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate)
+COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK
+\.
+
+-- The built-in options are handled in the same way of built-in formats.
+COPY copy_data TO stdout WITH (format 'test_format', delimiter ',');
+COPY copy_data TO stdout WITH (format 'test_format', quote '"');
+COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO
diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
new file mode 100644
index 00000000000..ca25832fcb0
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
@@ -0,0 +1,169 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_copy_custom_format.c
+ * Code for testing custom COPY format.
+ *
+ * Portions Copyright (c) 2026, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_copy_custom_format/test_copy_custom_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/copyapi.h"
+#include "commands/copy_state.h"
+#include "commands/defrem.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct TestCopyOptions
+{
+ int max_attributes;
+ bool disallow_freeze;
+} TestCopyOptions;
+
+static bool
+TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (t == NULL)
+ {
+ t = palloc0_object(TestCopyOptions);
+ opts->format_private_opts = (void *) t;
+ }
+
+ if (strcmp(option->defname, "max_attributes") == 0)
+ {
+ int val = defGetInt32(option);
+
+ if (val < 1)
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"max_attributes\" must be a positive integer"));
+
+ t->max_attributes = val;
+ return true;
+ }
+ else if (strcmp(option->defname, "disallow_freeze") == 0)
+ {
+ t->disallow_freeze = defGetBoolean(option);
+ return true;
+ }
+
+ return false;
+}
+
+static void
+TestCopyValidateOptions(CopyFormatOptions *opts, bool is_from)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (!t)
+ return;
+
+ if (t->disallow_freeze && opts->freeze)
+ ereport(ERROR,
+ errmsg("FREEZE cannot be used with \"disallow_freeze\" option"));
+}
+
+static void
+TestCopyFromInFunc(CopyFromState cstate, Oid atttypid, FmgrInfo *finfo, Oid *typioparam)
+{
+ ereport(NOTICE,
+ errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+check_max_attributes(CopyFormatOptions *opts, TupleDesc tupDesc)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (t != NULL && t->max_attributes > 0 && tupDesc->natts > t->max_attributes)
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("relation has %d columns, exceeds max_attributes %d",
+ tupDesc->natts, t->max_attributes));
+}
+
+static void
+TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ check_max_attributes(&cstate->opts, tupDesc);
+
+ ereport(NOTICE,
+ errmsg("CopyFromStart: the number of attributes of table: %d, the number of attributes to input: %d",
+ tupDesc->natts, list_length(cstate->attnumlist)));
+}
+
+static bool
+TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ ereport(NOTICE, errmsg("CopyFromOneRow"));
+
+ return false;
+}
+
+static void
+TestCopyFromEnd(CopyFromState cstate)
+{
+ ereport(NOTICE, errmsg("CopyFromEnd"));
+}
+
+static void
+TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo)
+{
+ ereport(NOTICE, errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+TestCopyToStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ check_max_attributes(&cstate->opts, tupDesc);
+
+ ereport(NOTICE,
+ errmsg("CopyToStart: the number of attributes of table: %d, the number of attributes to output: %d",
+ tupDesc->natts, list_length(cstate->attnumlist)));
+}
+
+static void
+TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid)));
+}
+
+static void
+TestCopyToEnd(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToEnd")));
+}
+
+static const CopyToRoutine TestCopyToRoutine = {
+ .CopyToOutFunc = TestCopyToOutFunc,
+ .CopyToStart = TestCopyToStart,
+ .CopyToOneRow = TestCopyToOneRow,
+ .CopyToEnd = TestCopyToEnd,
+};
+
+
+static const CopyFromRoutine TestCopyFromRoutine = {
+ .CopyFromInFunc = TestCopyFromInFunc,
+ .CopyFromStart = TestCopyFromStart,
+ .CopyFromOneRow = TestCopyFromOneRow,
+ .CopyFromEnd = TestCopyFromEnd,
+};
+
+void
+_PG_init(void)
+{
+ RegisterCopyCustomFormat("test_format",
+ &TestCopyToRoutine,
+ &TestCopyFromRoutine,
+ &TestCopyProcessOneOption,
+ &TestCopyValidateOptions);
+}
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 5263710e451..552669abc5f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3179,6 +3179,7 @@ Tcl_Obj
Tcl_Size
Tcl_Time
TempNamespaceStatus
+TestCopyOptions
TestDSMRegistryHashEntry
TestDSMRegistryStruct
TestDecodingData
--
2.54.0
[text/x-patch] v2-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch (5.6K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/4-v2-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch)
download | inline diff:
From c0e1e5986d10125b41f3fde10cdb3ba931db00b3 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 09:23:09 -0700
Subject: [PATCH v2 3/4] Add an hook for custom COPY format option validation.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
---
src/backend/commands/copy.c | 13 ++++++++++++-
src/backend/commands/copyapi.c | 10 ++++++++--
src/include/commands/copyapi.h | 15 +++++++++++++--
3 files changed, 33 insertions(+), 5 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 2fdba026ee0..45908d0c1e5 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -599,6 +599,7 @@ ProcessCopyOptions(ParseState *pstate,
*/
List *deferred_options = NIL;
ProcessOneOptionFn custom_process_option_fn = NULL;
+ ValidateOptionsFn custom_validate_options_fn = NULL;
char *custom_format_name = NULL;
/* Support external use for option sanity checking */
@@ -631,7 +632,8 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->format = COPY_FORMAT_JSON;
else if (GetCopyCustomFormatRoutines(fmt, &opts_out->to_routine,
&opts_out->from_routine,
- &custom_process_option_fn))
+ &custom_process_option_fn,
+ &custom_validate_options_fn))
{
opts_out->format = COPY_FORMAT_CUSTOM;
custom_format_name = fmt;
@@ -1126,6 +1128,15 @@ ProcessCopyOptions(ParseState *pstate,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY format \"%s\" cannot be used with COPY TO",
custom_format_name)));
+
+ /*
+ * Let the format validate its fully-parsed options as a whole. This
+ * runs even when no format-specific options were given, so a format
+ * can reject incompatible core options or enforce cross-option
+ * constraints.
+ */
+ if (custom_validate_options_fn != NULL)
+ custom_validate_options_fn(opts_out, is_from);
}
}
diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c
index 168efbcf30b..0bd1cb71af2 100644
--- a/src/backend/commands/copyapi.c
+++ b/src/backend/commands/copyapi.c
@@ -28,6 +28,7 @@ typedef struct CopyCustomFormatEntry
const CopyToRoutine *to_routine;
const CopyFromRoutine *from_routine;
ProcessOneOptionFn option_fn;
+ ValidateOptionsFn validate_fn;
} CopyCustomFormatEntry;
static CopyCustomFormatEntry *CopyCustomFormatArray = NULL;
@@ -56,7 +57,8 @@ is_builtin_copy_format(const char *name)
*/
void
RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
- const CopyFromRoutine *from, ProcessOneOptionFn option_fn)
+ const CopyFromRoutine *from, ProcessOneOptionFn option_fn,
+ ValidateOptionsFn validate_fn)
{
Assert(name != NULL && name[0] != '\0');
@@ -102,6 +104,7 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to;
CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from;
CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].validate_fn = validate_fn;
CopyCustomFormatsAssigned++;
}
@@ -111,7 +114,8 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
*/
bool
GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
- const CopyFromRoutine **from, ProcessOneOptionFn * option_fn)
+ const CopyFromRoutine **from, ProcessOneOptionFn * option_fn,
+ ValidateOptionsFn * validate_fn)
{
for (int i = 0; i < CopyCustomFormatsAssigned; i++)
{
@@ -123,6 +127,8 @@ GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
*from = CopyCustomFormatArray[i].from_routine;
if (option_fn)
*option_fn = CopyCustomFormatArray[i].option_fn;
+ if (validate_fn)
+ *validate_fn = CopyCustomFormatArray[i].validate_fn;
return true;
}
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 8eb5fe9c7dc..c47c89a858f 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -120,6 +120,15 @@ typedef struct CopyFromRoutine
typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from,
DefElem *option);
+/*
+ * Optional callback to validate a custom format's fully-parsed options as a
+ * whole. Invoked once from ProcessCopyOptions() after all options have been
+ * processed, so it can enforce cross-option constraints and reject
+ * incompatible core options. It runs even when no format-specific options were
+ * supplied. Reports problems with ereport().
+ */
+typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from);
+
/*
* Register a COPY format under 'name', mapping it to its TO and/or FROM
* routines and optional option/validation callbacks. Intended to be called
@@ -129,7 +138,8 @@ typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from,
*/
extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
const CopyFromRoutine *from,
- ProcessOneOptionFn option_fn);
+ ProcessOneOptionFn option_fn,
+ ValidateOptionsFn validate_fn);
/*
* Look up a previously registered custom format. Returns false if 'name' is
@@ -137,6 +147,7 @@ extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
*/
extern bool GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
const CopyFromRoutine **from,
- ProcessOneOptionFn * option_fn);
+ ProcessOneOptionFn * option_fn,
+ ValidateOptionsFn * validate_fn);
#endif /* COPYAPI_H */
--
2.54.0
[text/x-patch] v2-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch (26.4K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/5-v2-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch)
download | inline diff:
From 47aa927d65e483331f9b2010499b4c7f0bfb562c Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 11:50:27 -0700
Subject: [PATCH v2 1/4] Move Copy[From|To]StateData to copy_state.h.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
contrib/file_fdw/file_fdw.c | 2 +-
src/backend/commands/copyfrom.c | 5 +-
src/backend/commands/copyfromparse.c | 10 +-
src/backend/commands/copyto.c | 88 +-------
src/include/commands/copy.h | 2 +-
src/include/commands/copy_state.h | 253 +++++++++++++++++++++++
src/include/commands/copyfrom_internal.h | 165 +--------------
7 files changed, 273 insertions(+), 252 deletions(-)
create mode 100644 src/include/commands/copy_state.h
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 33a37d832ce..d152d05b92e 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -22,7 +22,7 @@
#include "catalog/pg_authid.h"
#include "catalog/pg_foreign_table.h"
#include "commands/copy.h"
-#include "commands/copyfrom_internal.h"
+#include "commands/copy_state.h"
#include "commands/defrem.h"
#include "commands/explain_format.h"
#include "commands/explain_state.h"
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 0087585b2c4..2c57b32f4de 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -30,6 +30,7 @@
#include "access/xact.h"
#include "catalog/namespace.h"
#include "commands/copyapi.h"
+#include "commands/copy_state.h"
#include "commands/copyfrom_internal.h"
#include "commands/progress.h"
#include "commands/trigger.h"
@@ -1732,7 +1733,7 @@ BeginCopyFrom(ParseState *pstate,
pg_encoding_to_char(GetDatabaseEncoding()))));
}
- cstate->copy_src = COPY_FILE; /* default */
+ cstate->copy_src = COPY_SOURCE_FILE; /* default */
cstate->whereClause = whereClause;
@@ -1861,7 +1862,7 @@ BeginCopyFrom(ParseState *pstate,
if (data_source_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_src = COPY_CALLBACK;
+ cstate->copy_src = COPY_SOURCE_CALLBACK;
cstate->data_source_cb = data_source_cb;
}
else if (pipe)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 65fd5a0ab4f..0ff5db4b62d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -184,7 +184,7 @@ ReceiveCopyBegin(CopyFromState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_src = COPY_FRONTEND;
+ cstate->copy_src = COPY_SOURCE_FRONTEND;
cstate->fe_msgbuf = makeStringInfo();
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -252,7 +252,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
switch (cstate->copy_src)
{
- case COPY_FILE:
+ case COPY_SOURCE_FILE:
pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ);
bytesread = fread(databuf, 1, maxread, cstate->copy_file);
pgstat_report_wait_end();
@@ -263,7 +263,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
if (bytesread == 0)
cstate->raw_reached_eof = true;
break;
- case COPY_FRONTEND:
+ case COPY_SOURCE_FRONTEND:
while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
{
int avail;
@@ -346,7 +346,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
bytesread += avail;
}
break;
- case COPY_CALLBACK:
+ case COPY_SOURCE_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
break;
}
@@ -1259,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (cstate->copy_src == COPY_SOURCE_FRONTEND)
{
int inbytes;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6755bb698de..ef2038c9a5d 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -23,8 +23,8 @@
#include "access/tupconvert.h"
#include "catalog/pg_inherits.h"
#include "commands/copyapi.h"
+#include "commands/copy_state.h"
#include "commands/progress.h"
-#include "executor/execdesc.h"
#include "executor/executor.h"
#include "executor/tuptable.h"
#include "funcapi.h"
@@ -42,76 +42,6 @@
#include "utils/snapmgr.h"
#include "utils/wait_event.h"
-/*
- * Represents the different dest cases we need to worry about at
- * the bottom level
- */
-typedef enum CopyDest
-{
- COPY_FILE, /* to file (or a piped program) */
- COPY_FRONTEND, /* to frontend */
- COPY_CALLBACK, /* to callback function */
-} CopyDest;
-
-/*
- * This struct contains all the state variables used throughout a COPY TO
- * operation.
- *
- * Multi-byte encodings: all supported client-side encodings encode multi-byte
- * characters by having the first byte's high bit set. Subsequent bytes of the
- * character can have the high bit not set. When scanning data in such an
- * encoding to look for a match to a single-byte (ie ASCII) character, we must
- * use the full pg_encoding_mblen() machinery to skip over multibyte
- * characters, else we might find a false match to a trailing byte. In
- * supported server encodings, there is no possibility of a false match, and
- * it's faster to make useless comparisons to trailing bytes than it is to
- * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
- * when we have to do it the hard way.
- */
-typedef struct CopyToStateData
-{
- /* format-specific routines */
- const CopyToRoutine *routine;
-
- /* low-level state data */
- CopyDest copy_dest; /* type of copy source/destination */
- FILE *copy_file; /* used if copy_dest == COPY_FILE */
- StringInfo fe_msgbuf; /* used for all dests during COPY TO */
-
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- bool json_row_delim_needed; /* need delimiter before next row */
- StringInfo json_buf; /* reusable buffer for JSON output,
- * initialized in BeginCopyTo */
- TupleDesc tupDesc; /* Descriptor for JSON output; for a column
- * list this is a projected descriptor */
- Datum *json_projvalues; /* pre-allocated projection values, or
- * NULL */
- bool *json_projnulls; /* pre-allocated projection nulls, or NULL */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
- List *partitions; /* OID list of partitions to copy data from */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- FmgrInfo *out_functions; /* lookup info for output functions */
- MemoryContext rowcontext; /* per-row evaluation context */
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
-
/* DestReceiver for COPY (query) TO */
typedef struct
{
@@ -559,7 +489,7 @@ SendCopyBegin(CopyToState cstate)
}
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->copy_dest = COPY_DEST_FRONTEND;
}
static void
@@ -606,7 +536,7 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
pgstat_report_wait_start(WAIT_EVENT_COPY_TO_WRITE);
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
@@ -642,11 +572,11 @@ CopySendEndOfRow(CopyToState cstate)
}
pgstat_report_wait_end();
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
- case COPY_CALLBACK:
+ case COPY_DEST_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
break;
}
@@ -667,7 +597,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
{
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
/* Default line termination depends on platform */
#ifndef WIN32
CopySendChar(cstate, '\n');
@@ -675,7 +605,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
CopySendString(cstate, "\r\n");
#endif
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* The FE/BE protocol uses \n as newline for all platforms */
CopySendChar(cstate, '\n');
break;
@@ -1135,12 +1065,12 @@ BeginCopyTo(ParseState *pstate,
/* See Multibyte encoding comment above */
cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->copy_dest = COPY_DEST_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->copy_dest = COPY_DEST_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index abecfe51098..5e710efff5b 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -99,7 +99,7 @@ typedef struct CopyFormatOptions
List *convert_select; /* list of column names (can be NIL) */
} CopyFormatOptions;
-/* These are private in commands/copy[from|to].c */
+/* These are defined in copy_state.h */
typedef struct CopyFromStateData *CopyFromState;
typedef struct CopyToStateData *CopyToState;
diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h
new file mode 100644
index 00000000000..52cbf5067eb
--- /dev/null
+++ b/src/include/commands/copy_state.h
@@ -0,0 +1,253 @@
+/*-------------------------------------------------------------------------
+ *
+ * copy_state.h
+ * prototypes for COPY TO/COPY FROM execution state.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994-5, Regents of the University of California
+ *
+ * src/include/commands/copy_state.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef COPY_STATE_H
+#define COPY_STATE_H
+
+#include "commands/copy.h"
+#include "commands/trigger.h"
+#include "executor/execdesc.h"
+#include "nodes/miscnodes.h"
+
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_SOURCE_FILE, /* from file (or a piped program) */
+ COPY_SOURCE_FRONTEND, /* from frontend */
+ COPY_SOURCE_CALLBACK, /* from callback function */
+} CopySource;
+
+/*
+ * Represents the end-of-line terminator type of the input
+ */
+typedef enum EolType
+{
+ EOL_UNKNOWN,
+ EOL_NL,
+ EOL_CR,
+ EOL_CRNL,
+} EolType;
+
+/*
+ * This struct contains all the state variables used throughout a COPY FROM
+ * operation.
+ */
+typedef struct CopyFromStateData
+{
+ /* format routine */
+ const struct CopyFromRoutine *routine;
+
+ /* low-level state data */
+ CopySource copy_src; /* type of copy source */
+ FILE *copy_file; /* used if copy_src == COPY_SOURCE_FILE */
+ StringInfo fe_msgbuf; /* used if copy_src == COPY_SOURCE_FRONTEND */
+
+ EolType eol_type; /* EOL type of input */
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ Oid conversion_proc; /* encoding conversion function */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ CopyFormatOptions opts;
+ bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /* these are just for error messages, see CopyFromErrorCallback */
+ const char *cur_relname; /* table name for error messages */
+ uint64 cur_lineno; /* line number for error messages */
+ const char *cur_attname; /* current att for error messages */
+ const char *cur_attval; /* current att value for error messages */
+ bool relname_only; /* don't output line number, att, etc. */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ AttrNumber num_defaults; /* count of att that are missing and have
+ * default value */
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+ ErrorSaveContext *escontext; /* soft error trapped during in_functions
+ * execution */
+ uint64 num_errors; /* total number of rows which contained soft
+ * errors */
+ int *defmap; /* array of default att numbers related to
+ * missing att */
+ ExprState **defexprs; /* array of default att expressions for all
+ * att */
+ bool *defaults; /* if DEFAULT marker was found for
+ * corresponding att */
+ bool simd_enabled; /* use SIMD to scan for special chars? */
+
+ /*
+ * True if the corresponding attribute's is a constrained domain. This
+ * will be populated only when ON_ERROR is SET_NULL, otherwise NULL.
+ */
+ bool *domain_with_constraint;
+
+ bool volatile_defexprs; /* is any of defexprs volatile? */
+ List *range_table; /* single element list of RangeTblEntry */
+ List *rteperminfos; /* single element list of RTEPermissionInfo */
+ ExprState *qualexpr;
+
+ TransitionCaptureState *transition_capture;
+
+ /*
+ * These variables are used to reduce overhead in COPY FROM.
+ *
+ * attribute_buf holds the separated, de-escaped text for each field of
+ * the current line. The CopyReadAttributes functions return arrays of
+ * pointers into this buffer. We avoid palloc/pfree overhead by re-using
+ * the buffer on each cycle.
+ *
+ * In binary COPY FROM, attribute_buf holds the binary data for the
+ * current field, but the usage is otherwise similar.
+ */
+ StringInfoData attribute_buf;
+
+ /* field raw data pointers found by COPY FROM */
+
+ int max_fields;
+ char **raw_fields;
+
+ /*
+ * Similarly, line_buf holds the whole input line being processed. The
+ * input cycle is first to read the whole line into line_buf, and then
+ * extract the individual attribute fields into attribute_buf. line_buf
+ * is preserved unmodified so that we can display it in error messages if
+ * appropriate. (In binary mode, line_buf is not used.)
+ */
+ StringInfoData line_buf;
+ bool line_buf_valid; /* contains the row being processed? */
+
+ /*
+ * input_buf holds input data, already converted to database encoding.
+ *
+ * In text mode, CopyReadLine parses this data sufficiently to locate line
+ * boundaries, then transfers the data to line_buf. We guarantee that
+ * there is a \0 at input_buf[input_buf_len] at all times. (In binary
+ * mode, input_buf is not used.)
+ *
+ * If encoding conversion is not required, input_buf is not a separate
+ * buffer but points directly to raw_buf. In that case, input_buf_len
+ * tracks the number of bytes that have been verified as valid in the
+ * database encoding, and raw_buf_len is the total number of bytes stored
+ * in the buffer.
+ */
+#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
+ char *input_buf;
+ int input_buf_index; /* next byte to process */
+ int input_buf_len; /* total # of bytes stored */
+ bool input_reached_eof; /* true if we reached EOF */
+ bool input_reached_error; /* true if a conversion error happened */
+ /* Shorthand for number of unconsumed bytes available in input_buf */
+#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
+
+ /*
+ * raw_buf holds raw input data read from the data source (file or client
+ * connection), not yet converted to the database encoding. Like with
+ * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
+ */
+#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
+ char *raw_buf;
+ int raw_buf_index; /* next byte to process */
+ int raw_buf_len; /* total # of bytes stored */
+ bool raw_reached_eof; /* true if we reached EOF */
+
+ /* Shorthand for number of unconsumed bytes available in raw_buf */
+#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
+
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyFromStateData;
+
+/*
+ * Represents the different dest cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopyDest
+{
+ COPY_DEST_FILE, /* to file (or a piped program) */
+ COPY_DEST_FRONTEND, /* to frontend */
+ COPY_DEST_CALLBACK, /* to callback function */
+} CopyDest;
+
+/*
+ * This struct contains all the state variables used throughout a COPY TO
+ * operation.
+ *
+ * Multi-byte encodings: all supported client-side encodings encode multi-byte
+ * characters by having the first byte's high bit set. Subsequent bytes of the
+ * character can have the high bit not set. When scanning data in such an
+ * encoding to look for a match to a single-byte (ie ASCII) character, we must
+ * use the full pg_encoding_mblen() machinery to skip over multibyte
+ * characters, else we might find a false match to a trailing byte. In
+ * supported server encodings, there is no possibility of a false match, and
+ * it's faster to make useless comparisons to trailing bytes than it is to
+ * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
+ * when we have to do it the hard way.
+ */
+typedef struct CopyToStateData
+{
+ /* format-specific routines */
+ const struct CopyToRoutine *routine;
+
+ /* low-level state data */
+ CopyDest copy_dest; /* type of copy source/destination */
+ FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */
+ StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ bool json_row_delim_needed; /* need delimiter before next row */
+ StringInfo json_buf; /* reusable buffer for JSON output,
+ * initialized in BeginCopyTo */
+ TupleDesc tupDesc; /* Descriptor for JSON output; for a column
+ * list this is a projected descriptor */
+ Datum *json_projvalues; /* pre-allocated projection values, or
+ * NULL */
+ bool *json_projnulls; /* pre-allocated projection nulls, or NULL */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+ List *partitions; /* OID list of partitions to copy data from */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+ MemoryContext rowcontext; /* per-row evaluation context */
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyToStateData;
+
+#endif /* COPY_STATE_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 9d3e244ee55..f7afade9a39 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -14,31 +14,7 @@
#ifndef COPYFROM_INTERNAL_H
#define COPYFROM_INTERNAL_H
-#include "commands/copy.h"
-#include "commands/trigger.h"
-#include "nodes/miscnodes.h"
-
-/*
- * Represents the different source cases we need to worry about at
- * the bottom level
- */
-typedef enum CopySource
-{
- COPY_FILE, /* from file (or a piped program) */
- COPY_FRONTEND, /* from frontend */
- COPY_CALLBACK, /* from callback function */
-} CopySource;
-
-/*
- * Represents the end-of-line terminator type of the input
- */
-typedef enum EolType
-{
- EOL_UNKNOWN,
- EOL_NL,
- EOL_CR,
- EOL_CRNL,
-} EolType;
+#include "commands/copy_state.h"
/*
* Represents the insert method to be used during COPY FROM.
@@ -52,145 +28,6 @@ typedef enum CopyInsertMethod
* ExecForeignBatchInsert only if valid */
} CopyInsertMethod;
-/*
- * This struct contains all the state variables used throughout a COPY FROM
- * operation.
- */
-typedef struct CopyFromStateData
-{
- /* format routine */
- const struct CopyFromRoutine *routine;
-
- /* low-level state data */
- CopySource copy_src; /* type of copy source */
- FILE *copy_file; /* used if copy_src == COPY_FILE */
- StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
-
- EolType eol_type; /* EOL type of input */
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- Oid conversion_proc; /* encoding conversion function */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
- bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /* these are just for error messages, see CopyFromErrorCallback */
- const char *cur_relname; /* table name for error messages */
- uint64 cur_lineno; /* line number for error messages */
- const char *cur_attname; /* current att for error messages */
- const char *cur_attval; /* current att value for error messages */
- bool relname_only; /* don't output line number, att, etc. */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- AttrNumber num_defaults; /* count of att that are missing and have
- * default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
- ErrorSaveContext *escontext; /* soft error trapped during in_functions
- * execution */
- uint64 num_errors; /* total number of rows which contained soft
- * errors */
- int *defmap; /* array of default att numbers related to
- * missing att */
- ExprState **defexprs; /* array of default att expressions for all
- * att */
- bool *defaults; /* if DEFAULT marker was found for
- * corresponding att */
- bool simd_enabled; /* use SIMD to scan for special chars? */
-
- /*
- * True if the corresponding attribute's is a constrained domain. This
- * will be populated only when ON_ERROR is SET_NULL, otherwise NULL.
- */
- bool *domain_with_constraint;
-
- bool volatile_defexprs; /* is any of defexprs volatile? */
- List *range_table; /* single element list of RangeTblEntry */
- List *rteperminfos; /* single element list of RTEPermissionInfo */
- ExprState *qualexpr;
-
- TransitionCaptureState *transition_capture;
-
- /*
- * These variables are used to reduce overhead in COPY FROM.
- *
- * attribute_buf holds the separated, de-escaped text for each field of
- * the current line. The CopyReadAttributes functions return arrays of
- * pointers into this buffer. We avoid palloc/pfree overhead by re-using
- * the buffer on each cycle.
- *
- * In binary COPY FROM, attribute_buf holds the binary data for the
- * current field, but the usage is otherwise similar.
- */
- StringInfoData attribute_buf;
-
- /* field raw data pointers found by COPY FROM */
-
- int max_fields;
- char **raw_fields;
-
- /*
- * Similarly, line_buf holds the whole input line being processed. The
- * input cycle is first to read the whole line into line_buf, and then
- * extract the individual attribute fields into attribute_buf. line_buf
- * is preserved unmodified so that we can display it in error messages if
- * appropriate. (In binary mode, line_buf is not used.)
- */
- StringInfoData line_buf;
- bool line_buf_valid; /* contains the row being processed? */
-
- /*
- * input_buf holds input data, already converted to database encoding.
- *
- * In text mode, CopyReadLine parses this data sufficiently to locate line
- * boundaries, then transfers the data to line_buf. We guarantee that
- * there is a \0 at input_buf[input_buf_len] at all times. (In binary
- * mode, input_buf is not used.)
- *
- * If encoding conversion is not required, input_buf is not a separate
- * buffer but points directly to raw_buf. In that case, input_buf_len
- * tracks the number of bytes that have been verified as valid in the
- * database encoding, and raw_buf_len is the total number of bytes stored
- * in the buffer.
- */
-#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
- char *input_buf;
- int input_buf_index; /* next byte to process */
- int input_buf_len; /* total # of bytes stored */
- bool input_reached_eof; /* true if we reached EOF */
- bool input_reached_error; /* true if a conversion error happened */
- /* Shorthand for number of unconsumed bytes available in input_buf */
-#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
-
- /*
- * raw_buf holds raw input data read from the data source (file or client
- * connection), not yet converted to the database encoding. Like with
- * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
- */
-#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
- char *raw_buf;
- int raw_buf_index; /* next byte to process */
- int raw_buf_len; /* total # of bytes stored */
- bool raw_reached_eof; /* true if we reached EOF */
-
- /* Shorthand for number of unconsumed bytes available in raw_buf */
-#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
-
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
-
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
--
2.54.0
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2026-06-23 05:41 ` Sutou Kouhei <[email protected]>
2026-06-24 01:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Sutou Kouhei @ 2026-06-23 05:41 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
Thanks for restarting this.
In <CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 22 Jun 2026 18:06:07 -0700,
Masahiko Sawada <[email protected]> wrote:
> After more thought, I'd like to keep the custom-format changes to the
> bare minimum and not disturb the existing built-in format processing.
+1
> Updated patches attached:
>
> - 0001 moves CopyFromStateData and CopyToStateData to a new
> copy_state.h, so extensions can implement their routines without
> including the *_internal.h headers. It also drops file_fdw.c's
> dependency on copyfrom_internal.h.
+1
> - 0002 introduces the registration API and the opaque per-format
> pointer in both structs.
> --- /dev/null
> +++ b/src/backend/commands/copyapi.c
> +bool
> +GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
> + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn)
How about returning CopyCustomFormatEntry instead? The
function name is "Get...Routines" but it also returns
ProcessOneOptionFn. "Get...Routines" is a bit strange.
> --- a/src/include/commands/copyapi.h
> +++ b/src/include/commands/copyapi.h
> @@ -102,4 +103,40 @@ typedef struct CopyFromRoutine
> ...
> +typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from,
> + DefElem *option);
How about adding "Copy" keyword to the type name such as
"ProcessOneCopyOptionFn" because this is only for COPY format?
> --- a/src/include/commands/copy.h
> +++ b/src/include/commands/copy.h
> @@ -58,7 +58,16 @@ typedef enum CopyFormat
> ...
> +#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM)
How about renaming this to CopyFormatIsBuiltin() or
something? "...Builtins" is a bit strange because this
returns a boolean.
> - 0003 adds a callback to validate the COPY options as a whole, called
> after all options are processed.
> --- a/src/include/commands/copyapi.h
> +++ b/src/include/commands/copyapi.h
> @@ -120,6 +120,15 @@ typedef struct CopyFromRoutine
> ...
> +typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from);
How about adding "Copy" keyword like "ValidateCopyOptionsFn"?
> - 0004 adds the regression tests.
> --- /dev/null
> +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
> @@ -0,0 +1,169 @@
> ...
> +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option)
> +{
> + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
> +
> + if (t == NULL)
> + {
> + t = palloc0_object(TestCopyOptions);
> + opts->format_private_opts = (void *) t;
> + }
This is not a blocker but we may want to add
InitializeCopyOptions callback for this.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 05:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2026-06-24 01:15 ` Masahiko Sawada <[email protected]>
2026-06-26 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2026-06-24 01:15 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jun 22, 2026 at 10:41 PM Sutou Kouhei <[email protected]> wrote:
>
> > - 0002 introduces the registration API and the opaque per-format
> > pointer in both structs.
>
> > --- /dev/null
> > +++ b/src/backend/commands/copyapi.c
>
> > +bool
> > +GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to,
> > + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn)
>
> How about returning CopyCustomFormatEntry instead? The
> function name is "Get...Routines" but it also returns
> ProcessOneOptionFn. "Get...Routines" is a bit strange.
Agreed.
>
> > --- a/src/include/commands/copyapi.h
> > +++ b/src/include/commands/copyapi.h
>
> > @@ -102,4 +103,40 @@ typedef struct CopyFromRoutine
> > ...
> > +typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from,
> > + DefElem *option);
>
> How about adding "Copy" keyword to the type name such as
> "ProcessOneCopyOptionFn" because this is only for COPY format?
Agreed.
>
> > --- a/src/include/commands/copy.h
> > +++ b/src/include/commands/copy.h
>
> > @@ -58,7 +58,16 @@ typedef enum CopyFormat
> > ...
> > +#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM)
>
> How about renaming this to CopyFormatIsBuiltin() or
> something? "...Builtins" is a bit strange because this
> returns a boolean.
Agreed.
>
> > - 0003 adds a callback to validate the COPY options as a whole, called
> > after all options are processed.
>
> > --- a/src/include/commands/copyapi.h
> > +++ b/src/include/commands/copyapi.h
> > @@ -120,6 +120,15 @@ typedef struct CopyFromRoutine
> > ...
> > +typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from);
>
> How about adding "Copy" keyword like "ValidateCopyOptionsFn"?
Agreed.
>
> > - 0004 adds the regression tests.
>
> > --- /dev/null
> > +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
>
> > @@ -0,0 +1,169 @@
> > ...
> > +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option)
> > +{
> > + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
> > +
> > + if (t == NULL)
> > + {
> > + t = palloc0_object(TestCopyOptions);
> > + opts->format_private_opts = (void *) t;
> > + }
>
> This is not a blocker but we may want to add
> InitializeCopyOptions callback for this.
It would save just a few lines. It might be worth having the
initialization callback if it would enable extensions to do what
cannot be done with the current proposed callbacks.
Thank you for reviewing the patches! I've attached updated patches.
I'll verify that the new API works well with an experimental custom
copy format extension.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-patch] v3-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch (3.6K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/2-v3-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch)
download | inline diff:
From 43c06a26a14c914f6bc3d72d01accd95e3b0f8e8 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 09:23:09 -0700
Subject: [PATCH v3 3/4] Add an hook for custom COPY format option validation.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
---
src/backend/commands/copy.c | 9 +++++++++
src/backend/commands/copyapi.c | 4 +++-
src/include/commands/copyapi.h | 14 +++++++++++++-
3 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 0fda54905cd..dab523c68db 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1119,6 +1119,15 @@ ProcessCopyOptions(ParseState *pstate,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY format \"%s\" cannot be used with COPY TO",
opts_out->custom_format_ent->name)));
+
+ /*
+ * Let the format validate its fully-parsed options as a whole. This
+ * runs even when no format-specific options were given, so a format
+ * can reject incompatible core options or enforce cross-option
+ * constraints.
+ */
+ if (opts_out->custom_format_ent->validate_fn != NULL)
+ opts_out->custom_format_ent->validate_fn(opts_out, is_from);
}
}
diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c
index f007d73e867..29fb98ff4c2 100644
--- a/src/backend/commands/copyapi.c
+++ b/src/backend/commands/copyapi.c
@@ -48,7 +48,8 @@ is_builtin_copy_format(const char *name)
*/
void
RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
- const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn)
+ const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn,
+ ValidateCopyOptionsFn validate_fn)
{
Assert(name != NULL && name[0] != '\0');
@@ -94,6 +95,7 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to;
CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from;
CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].validate_fn = validate_fn;
CopyCustomFormatsAssigned++;
}
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 1b323b23bba..f1924424df8 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -117,6 +117,15 @@ typedef struct CopyFromRoutine
typedef bool (*ProcessOneCopyOptionFn) (CopyFormatOptions *opts, bool is_from,
DefElem *option);
+/*
+ * Optional callback to validate a custom format's fully-parsed options as a
+ * whole. Invoked once from ProcessCopyOptions() after all options have been
+ * processed, so it can enforce cross-option constraints and reject
+ * incompatible core options. It runs even when no format-specific options were
+ * supplied. Reports problems with ereport().
+ */
+typedef void (*ValidateCopyOptionsFn) (CopyFormatOptions *opts, bool is_from);
+
/*
* Sturct to store the registered custom format information.
*/
@@ -126,11 +135,14 @@ typedef struct CopyCustomFormatEntry
const CopyToRoutine *to_routine;
const CopyFromRoutine *from_routine;
ProcessOneCopyOptionFn option_fn;
+ ValidateCopyOptionsFn validate_fn;
} CopyCustomFormatEntry;
extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
const CopyFromRoutine *from,
- ProcessOneCopyOptionFn option_fn);
+ ProcessOneCopyOptionFn option_fn,
+ ValidateCopyOptionsFn validate_fn);
+
extern const CopyCustomFormatEntry *GetCopyCustomFormatRoutines(const char *name);
#endif /* COPYAPI_H */
--
2.54.0
[text/x-patch] v3-0002-Allow-extensions-to-register-custom-format-to-COP.patch (18.1K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/3-v3-0002-Allow-extensions-to-register-custom-format-to-COP.patch)
download | inline diff:
From 4b1bbfb28daaa8a177b3a802572ae930fc15d199 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 09:21:51 -0700
Subject: [PATCH v3 2/4] Allow extensions to register custom format to COPY TO
and COPY FROM.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
---
src/backend/commands/Makefile | 1 +
src/backend/commands/copy.c | 90 ++++++++++++++++++++---
src/backend/commands/copyapi.c | 114 ++++++++++++++++++++++++++++++
src/backend/commands/copyfrom.c | 4 +-
src/backend/commands/copyto.c | 4 +-
src/backend/commands/meson.build | 1 +
src/include/commands/copy.h | 18 +++++
src/include/commands/copy_state.h | 6 ++
src/include/commands/copyapi.h | 31 ++++++++
src/tools/pgindent/typedefs.list | 1 +
10 files changed, 259 insertions(+), 11 deletions(-)
create mode 100644 src/backend/commands/copyapi.c
diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile
index 5b9d084977e..17b7aa08b55 100644
--- a/src/backend/commands/Makefile
+++ b/src/backend/commands/Makefile
@@ -23,6 +23,7 @@ OBJS = \
constraint.o \
conversioncmds.o \
copy.o \
+ copyapi.o \
copyfrom.o \
copyfromparse.o \
copyto.o \
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 003b70852bb..0fda54905cd 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -23,6 +23,7 @@
#include "access/xact.h"
#include "catalog/pg_authid.h"
#include "commands/copy.h"
+#include "commands/copyapi.h"
#include "commands/defrem.h"
#include "executor/executor.h"
#include "mb/pg_wchar.h"
@@ -592,6 +593,12 @@ ProcessCopyOptions(ParseState *pstate,
bool force_array_specified = false;
ListCell *option;
+ /*
+ * Options not recognized by core are collected here and, once the format
+ * is known, either handed to a custom format's option parser or rejected.
+ */
+ List *deferred_options = NIL;
+
/* Support external use for option sanity checking */
if (opts_out == NULL)
opts_out = palloc0_object(CopyFormatOptions);
@@ -620,6 +627,8 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->format = COPY_FORMAT_BINARY;
else if (strcmp(fmt, "json") == 0)
opts_out->format = COPY_FORMAT_JSON;
+ else if ((opts_out->custom_format_ent = GetCopyCustomFormatRoutines(fmt)) != NULL)
+ opts_out->format = COPY_FORMAT_CUSTOM;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -775,11 +784,54 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->reject_limit = defGetCopyRejectLimitOption(defel);
}
else
+ {
+ /*
+ * Not a core option. Defer the check to after the loop as it may
+ * belong to a custom format whose "format" option has not been
+ * seen yet.
+ */
+ deferred_options = lappend(deferred_options, defel);
+ }
+ }
+
+ /*
+ * Now that the format and every option have been seen, resolve the
+ * deferred options.
+ */
+ if (deferred_options != NIL)
+ {
+ /*
+ * For a custom format, they belong to the handler; for any built-in
+ * (including the default) an unrecognized option is an error,
+ * preserving the historical behavior relied on by external callers
+ * such as file_fdw.
+ */
+ if (opts_out->format != COPY_FORMAT_CUSTOM || opts_out->custom_format_ent->option_fn == NULL)
+ {
+ DefElem *defel = linitial_node(DefElem, deferred_options);
+
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("option \"%s\" not recognized",
defel->defname),
parser_errposition(pstate, defel->location)));
+ }
+
+ /*
+ * Hand each option core did not recognize to the format's per-option
+ * callback. Anything the format does not claim (or any option at all
+ * if it has no callback) is an error, so an unrecognized option
+ * always fails here.
+ */
+ foreach_node(DefElem, opt, deferred_options)
+ {
+ if (!opts_out->custom_format_ent->option_fn(opts_out, is_from, opt))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("COPY format \"%s\" does not accept option \"%s\"",
+ opts_out->custom_format_ent->name, opt->defname),
+ parser_errposition(pstate, opt->location)));
+ }
}
/*
@@ -869,7 +921,7 @@ ProcessCopyOptions(ParseState *pstate,
* future-proofing. Likewise we disallow all digits though only octal
* digits are actually dangerous.
*/
- if (opts_out->format != COPY_FORMAT_CSV &&
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
opts_out->delim[0]) != NULL)
ereport(ERROR,
@@ -888,7 +940,8 @@ ProcessCopyOptions(ParseState *pstate,
: errmsg("cannot specify %s in JSON mode", "HEADER"));
/* Check quote */
- if (opts_out->format != COPY_FORMAT_CSV && opts_out->quote != NULL)
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ opts_out->quote != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -905,7 +958,8 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("COPY delimiter and quote must be different")));
/* Check escape */
- if (opts_out->format != COPY_FORMAT_CSV && opts_out->escape != NULL)
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ opts_out->escape != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -917,7 +971,8 @@ ProcessCopyOptions(ParseState *pstate,
errmsg("COPY escape must be a single one-byte character")));
/* Check force_quote */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_quote || opts_out->force_quote_all))
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_quote || opts_out->force_quote_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -931,8 +986,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY FROM")));
/* Check force_notnull */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_notnull != NIL ||
- opts_out->force_notnull_all))
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_notnull != NIL || opts_out->force_notnull_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -947,8 +1002,8 @@ ProcessCopyOptions(ParseState *pstate,
"COPY TO")));
/* Check force_null */
- if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_null != NIL ||
- opts_out->force_null_all))
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV &&
+ (opts_out->force_null != NIL || opts_out->force_null_all))
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
/*- translator: %s is the name of a COPY option, e.g. ON_ERROR */
@@ -995,7 +1050,8 @@ ProcessCopyOptions(ParseState *pstate,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY %s is not supported for %s", "FORMAT JSON", "COPY FROM"));
- if (opts_out->format != COPY_FORMAT_JSON && opts_out->force_array)
+ if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_JSON &&
+ opts_out->force_array)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY %s can only be used with JSON mode", "FORCE_ARRAY"));
@@ -1048,6 +1104,22 @@ ProcessCopyOptions(ParseState *pstate,
* ON_ERROR, third is the value of the COPY option, e.g. IGNORE */
errmsg("COPY %s requires %s to be set to %s",
"REJECT_LIMIT", "ON_ERROR", "IGNORE")));
+
+ /* Check custom format routines */
+ if (opts_out->format == COPY_FORMAT_CUSTOM)
+ {
+ if (is_from && opts_out->custom_format_ent->from_routine == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY format \"%s\" cannot be used with COPY FROM",
+ opts_out->custom_format_ent->name)));
+
+ if (!is_from && opts_out->custom_format_ent->to_routine == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("COPY format \"%s\" cannot be used with COPY TO",
+ opts_out->custom_format_ent->name)));
+ }
}
/*
diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c
new file mode 100644
index 00000000000..f007d73e867
--- /dev/null
+++ b/src/backend/commands/copyapi.c
@@ -0,0 +1,114 @@
+/*-------------------------------------------------------------------------
+ *
+ * copyapi.c
+ * Registry for pluggable COPY TO/FROM format handlers.
+ *
+ * The built-in formats (text, csv, binary, json) are dispatched directly by
+ * the COPY engine. Extensions can provide additional formats by registering
+ * a CopyToRoutine and/or CopyFromRoutine under a name from their _PG_init();
+ * ProcessCopyOptions() then resolves "COPY ... (FORMAT 'name')" against this
+ * registry.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/commands/copyapi.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "commands/copyapi.h"
+#include "utils/memutils.h"
+
+static CopyCustomFormatEntry *CopyCustomFormatArray = NULL;
+static int CopyCustomFormatsAssigned = 0;
+static int CopyCustomFormatsAllocated = 0;
+
+/* Is 'name' one of the built-in format keywords? */
+static bool
+is_builtin_copy_format(const char *name)
+{
+ return (strcmp(name, "text") == 0 ||
+ strcmp(name, "csv") == 0 ||
+ strcmp(name, "binary") == 0 ||
+ strcmp(name, "json") == 0);
+}
+
+/*
+ * Register a custom COPY format. Intended to be called from an extension's
+ * _PG_init(). Either routine may be NULL if the format does not support that
+ * direction (but not both).
+ *
+ * 'option_fn' may also be NULL if the format takes no format-specific options.
+ *
+ * 'name' is assumed to be a constant string or allocated in storage that will
+ * never be freed; it is stored by reference.
+ */
+void
+RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
+ const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn)
+{
+ Assert(name != NULL && name[0] != '\0');
+
+ /* Must support at least one direction */
+ Assert(to != NULL || from != NULL);
+
+ Assert(to == NULL ||
+ (to->CopyToStart != NULL && to->CopyToOneRow != NULL &&
+ to->CopyToEnd != NULL));
+ Assert(from == NULL ||
+ (from->CopyFromStart != NULL && from->CopyFromOneRow != NULL &&
+ from->CopyFromEnd != NULL));
+
+ /* Check if it's already used by built-in format names */
+ if (is_builtin_copy_format(name))
+ elog(ERROR, "COPY format \"%s\" is a built-in format name", name);
+
+ /* Reject a duplicate registration. */
+ for (int i = 0; i < CopyCustomFormatsAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].name, name) == 0)
+ elog(ERROR, "COPY format \"%s\" is already registered", name);
+ }
+
+ /* Create the array on first use; it must outlive the current context. */
+ if (CopyCustomFormatArray == NULL)
+ {
+ CopyCustomFormatsAllocated = 16;
+ CopyCustomFormatArray = (CopyCustomFormatEntry *)
+ MemoryContextAlloc(TopMemoryContext,
+ CopyCustomFormatsAllocated * sizeof(CopyCustomFormatEntry));
+ }
+
+ /* Expand if full. */
+ if (CopyCustomFormatsAssigned >= CopyCustomFormatsAllocated)
+ {
+ CopyCustomFormatsAllocated *= 2;
+ CopyCustomFormatArray = (CopyCustomFormatEntry *)
+ repalloc_array(CopyCustomFormatArray, CopyCustomFormatEntry, CopyCustomFormatsAllocated);
+ }
+
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].name = name;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from;
+ CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn;
+ CopyCustomFormatsAssigned++;
+}
+
+/*
+ * Look up a previously registered custom format. Returns NULL if 'name' is
+ * not registered.
+ */
+const CopyCustomFormatEntry *
+GetCopyCustomFormatRoutines(const char *name)
+{
+ for (int i = 0; i < CopyCustomFormatsAssigned; i++)
+ {
+ if (strcmp(CopyCustomFormatArray[i].name, name) == 0)
+ return &(CopyCustomFormatArray[i]);
+ }
+
+ return NULL;
+}
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 2c57b32f4de..ff4fa718e56 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -158,7 +158,9 @@ static const CopyFromRoutine CopyFromRoutineBinary = {
static const CopyFromRoutine *
CopyFromGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->format == COPY_FORMAT_CSV)
+ if (opts->format == COPY_FORMAT_CUSTOM)
+ return opts->custom_format_ent->from_routine;
+ else if (opts->format == COPY_FORMAT_CSV)
return &CopyFromRoutineCSV;
else if (opts->format == COPY_FORMAT_BINARY)
return &CopyFromRoutineBinary;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index ef2038c9a5d..72cc1ac8d8d 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -130,7 +130,9 @@ static const CopyToRoutine CopyToRoutineBinary = {
static const CopyToRoutine *
CopyToGetRoutine(const CopyFormatOptions *opts)
{
- if (opts->format == COPY_FORMAT_CSV)
+ if (opts->format == COPY_FORMAT_CUSTOM)
+ return opts->custom_format_ent->to_routine;
+ else if (opts->format == COPY_FORMAT_CSV)
return &CopyToRoutineCSV;
else if (opts->format == COPY_FORMAT_BINARY)
return &CopyToRoutineBinary;
diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build
index 9f258d566eb..d98273da67e 100644
--- a/src/backend/commands/meson.build
+++ b/src/backend/commands/meson.build
@@ -11,6 +11,7 @@ backend_sources += files(
'constraint.c',
'conversioncmds.c',
'copy.c',
+ 'copyapi.c',
'copyfrom.c',
'copyfromparse.c',
'copyto.c',
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 5e710efff5b..77c652818cb 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -58,7 +58,16 @@ typedef enum CopyFormat
COPY_FORMAT_BINARY,
COPY_FORMAT_CSV,
COPY_FORMAT_JSON,
+ COPY_FORMAT_CUSTOM, /* format provided by an extension */
} CopyFormat;
+#define CopyFormatIsBuiltins(format) ((format) != COPY_FORMAT_CUSTOM)
+
+/*
+ * Full definitions live in commands/copyapi.h, which includes this header;
+ * CopyFormatOptions only needs to hold pointers to the resolved routines.
+ */
+struct CopyToRoutine;
+struct CopyFromRoutine;
/*
* A struct to hold COPY options, in a parsed form. All of these are related
@@ -97,6 +106,15 @@ typedef struct CopyFormatOptions
CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */
int64 reject_limit; /* maximum tolerable number of errors */
List *convert_select; /* list of column names (can be NIL) */
+
+ /*
+ * Resolved handler for a custom format. The directoin not in use may be
+ * NULL. For built-in formats these are unused.
+ */
+ const struct CopyCustomFormatEntry *custom_format_ent;
+
+ /* Custom format private option data */
+ void *format_private_opts;
} CopyFormatOptions;
/* These are defined in copy_state.h */
diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h
index 52cbf5067eb..6c5defbf4ee 100644
--- a/src/include/commands/copy_state.h
+++ b/src/include/commands/copy_state.h
@@ -178,6 +178,9 @@ typedef struct CopyFromStateData
#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* Custom format private data to store the state */
+ void *format_private;
} CopyFromStateData;
/*
@@ -248,6 +251,9 @@ typedef struct CopyToStateData
FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* Custom format private data to store the state */
+ void *format_private;
} CopyToStateData;
#endif /* COPY_STATE_H */
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 398e7a78bb3..1b323b23bba 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,6 +14,7 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "commands/copy_state.h"
#include "commands/copy.h"
/*
@@ -102,4 +103,34 @@ typedef struct CopyFromRoutine
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
+/*
+ * Optional callback to process one format-specific COPY option. Invoked
+ * from ProcessCopyOptions() once per option that core did not recognize, after
+ * every core option has been parsed (so 'opts' is fully populated).
+ *
+ * Returns true if the option belongs to the format and is valid. Returns false
+ * if the option is not one the format recognizes, in which case core raises the
+ * "not accepted" error; thus an unrecognized option always errors, whether or
+ * not the format supplies this callback. For a recognized option with an invalid
+ * value, the callback should ereport() itself.
+ */
+typedef bool (*ProcessOneCopyOptionFn) (CopyFormatOptions *opts, bool is_from,
+ DefElem *option);
+
+/*
+ * Sturct to store the registered custom format information.
+ */
+typedef struct CopyCustomFormatEntry
+{
+ const char *name; /* constant string; never freed (see below) */
+ const CopyToRoutine *to_routine;
+ const CopyFromRoutine *from_routine;
+ ProcessOneCopyOptionFn option_fn;
+} CopyCustomFormatEntry;
+
+extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
+ const CopyFromRoutine *from,
+ ProcessOneCopyOptionFn option_fn);
+extern const CopyCustomFormatEntry *GetCopyCustomFormatRoutines(const char *name);
+
#endif /* COPYAPI_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 1969d467c1d..5263710e451 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -540,6 +540,7 @@ ConvProcInfo
ConversionLocation
ConvertRowtypeExpr
CookedConstraint
+CopyCustomFormatEntry
CopyDest
CopyFormat
CopyFormatOptions
--
2.54.0
[text/x-patch] v3-0004-Add-test-module-for-COPY-custom-format.patch (17.3K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/4-v3-0004-Add-test-module-for-COPY-custom-format.patch)
download | inline diff:
From b5ff7ebea3141625b8699c85a3bf15972b1e1537 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 13:14:31 -0700
Subject: [PATCH v3 4/4] Add test module for COPY custom format.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../modules/test_copy_custom_format/Makefile | 20 +++
.../expected/test_copy_custom_format.out | 105 +++++++++++
.../test_copy_custom_format/meson.build | 32 ++++
.../sql/test_copy_custom_format.sql | 32 ++++
.../test_copy_custom_format.c | 169 ++++++++++++++++++
src/tools/pgindent/typedefs.list | 1 +
8 files changed, 361 insertions(+)
create mode 100644 src/test/modules/test_copy_custom_format/Makefile
create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
create mode 100644 src/test/modules/test_copy_custom_format/meson.build
create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 0a74ab5c86f..6dcb66174f5 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -23,6 +23,7 @@ SUBDIRS = \
test_cloexec \
test_checksums \
test_copy_callbacks \
+ test_copy_custom_format \
test_custom_rmgrs \
test_custom_stats \
test_custom_types \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 4bca42bb370..adfa413fe58 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -23,6 +23,7 @@ subdir('test_bloomfilter')
subdir('test_cloexec')
subdir('test_checksums')
subdir('test_copy_callbacks')
+subdir('test_copy_custom_format')
subdir('test_cplusplusext')
subdir('test_custom_rmgrs')
subdir('test_custom_stats')
diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile
new file mode 100644
index 00000000000..68a2a04ff09
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/Makefile
@@ -0,0 +1,20 @@
+# src/test/modules/test_copy_custom_format/Makefile
+
+MODULE_big = test_copy_custom_format
+OBJS = \
+ $(WIN32RES) \
+ test_copy_custom_format.o
+PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT"
+
+REGRESS = test_copy_custom_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_copy_custom_format
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
new file mode 100644
index 00000000000..817ca3fa60f
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out
@@ -0,0 +1,105 @@
+LOAD 'test_copy_custom_format';
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789);
+COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 2
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToStart: the number of attributes of table: 1, the number of attributes to output: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToOneRow: the number of valid values: 1
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized
+ERROR: COPY format "nonexistent" not recognized
+LINE 1: COPY copy_data TO stdout WITH (format 'nonexistent');
+ ^
+COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting
+ERROR: conflicting or redundant options
+LINE 1: COPY copy_data TO stdout WITH (format 'text', format 'csv');
+ ^
+COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR
+ERROR: COPY format "test_format" does not accept option "bogus"
+LINE 1: ...PY copy_data TO stdout WITH (format 'test_format', bogus 1);
+ ^
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+ERROR: relation has 3 columns, exceeds max_attributes 2
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive
+ERROR: "max_attributes" must be a positive integer
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR
+ERROR: "max_attributes" must be a positive integer
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required
+ERROR: max_attributes requires an integer value
+COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate)
+ERROR: FREEZE cannot be used with "disallow_freeze" option
+COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK
+NOTICE: CopyFromInFunc: attribute: smallint
+NOTICE: CopyFromInFunc: attribute: integer
+NOTICE: CopyFromInFunc: attribute: bigint
+NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
+-- The built-in options are handled in the same way of built-in formats.
+COPY copy_data TO stdout WITH (format 'test_format', delimiter ',');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', quote '"');
+NOTICE: CopyToOutFunc: attribute: smallint
+NOTICE: CopyToOutFunc: attribute: integer
+NOTICE: CopyToOutFunc: attribute: bigint
+NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToOneRow: the number of valid values: 3
+NOTICE: CopyToEnd
+COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO
+ERROR: COPY FREEZE cannot be used with COPY TO
diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build
new file mode 100644
index 00000000000..a231ed57649
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/meson.build
@@ -0,0 +1,32 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+test_copy_custom_format_sources = files(
+'test_copy_custom_format.c',
+)
+
+if host_system == 'windows'
+ test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_copy_custom_format',
+ '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',])
+endif
+
+test_copy_custom_format = shared_module('test_copy_custom_format',
+ test_copy_custom_format_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_copy_custom_format
+
+tests += {
+ 'name': 'test_copy_custom_format',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_copy_custom_format',
+ ],
+ # Disabled because these tests require
+ # "shared_preload_libraries=test_custom_copy_format", which typical
+ # runningcheck users do not have (e.g. buildfarm clients).
+ 'runningcheck': false,
+ },
+}
diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
new file mode 100644
index 00000000000..59f58fa55a2
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql
@@ -0,0 +1,32 @@
+LOAD 'test_copy_custom_format';
+
+CREATE TABLE copy_data (a smallint, b integer, c bigint);
+INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789);
+
+COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End
+COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End
+\.
+
+COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2
+COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1
+
+COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized
+COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting
+
+COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR
+
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR
+COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required
+
+COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate)
+COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK
+\.
+
+-- The built-in options are handled in the same way of built-in formats.
+COPY copy_data TO stdout WITH (format 'test_format', delimiter ',');
+COPY copy_data TO stdout WITH (format 'test_format', quote '"');
+COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO
diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
new file mode 100644
index 00000000000..ca25832fcb0
--- /dev/null
+++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c
@@ -0,0 +1,169 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_copy_custom_format.c
+ * Code for testing custom COPY format.
+ *
+ * Portions Copyright (c) 2026, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_copy_custom_format/test_copy_custom_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/copyapi.h"
+#include "commands/copy_state.h"
+#include "commands/defrem.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct TestCopyOptions
+{
+ int max_attributes;
+ bool disallow_freeze;
+} TestCopyOptions;
+
+static bool
+TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (t == NULL)
+ {
+ t = palloc0_object(TestCopyOptions);
+ opts->format_private_opts = (void *) t;
+ }
+
+ if (strcmp(option->defname, "max_attributes") == 0)
+ {
+ int val = defGetInt32(option);
+
+ if (val < 1)
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"max_attributes\" must be a positive integer"));
+
+ t->max_attributes = val;
+ return true;
+ }
+ else if (strcmp(option->defname, "disallow_freeze") == 0)
+ {
+ t->disallow_freeze = defGetBoolean(option);
+ return true;
+ }
+
+ return false;
+}
+
+static void
+TestCopyValidateOptions(CopyFormatOptions *opts, bool is_from)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (!t)
+ return;
+
+ if (t->disallow_freeze && opts->freeze)
+ ereport(ERROR,
+ errmsg("FREEZE cannot be used with \"disallow_freeze\" option"));
+}
+
+static void
+TestCopyFromInFunc(CopyFromState cstate, Oid atttypid, FmgrInfo *finfo, Oid *typioparam)
+{
+ ereport(NOTICE,
+ errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+check_max_attributes(CopyFormatOptions *opts, TupleDesc tupDesc)
+{
+ TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts;
+
+ if (t != NULL && t->max_attributes > 0 && tupDesc->natts > t->max_attributes)
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("relation has %d columns, exceeds max_attributes %d",
+ tupDesc->natts, t->max_attributes));
+}
+
+static void
+TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ check_max_attributes(&cstate->opts, tupDesc);
+
+ ereport(NOTICE,
+ errmsg("CopyFromStart: the number of attributes of table: %d, the number of attributes to input: %d",
+ tupDesc->natts, list_length(cstate->attnumlist)));
+}
+
+static bool
+TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ ereport(NOTICE, errmsg("CopyFromOneRow"));
+
+ return false;
+}
+
+static void
+TestCopyFromEnd(CopyFromState cstate)
+{
+ ereport(NOTICE, errmsg("CopyFromEnd"));
+}
+
+static void
+TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo)
+{
+ ereport(NOTICE, errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid)));
+}
+
+static void
+TestCopyToStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ check_max_attributes(&cstate->opts, tupDesc);
+
+ ereport(NOTICE,
+ errmsg("CopyToStart: the number of attributes of table: %d, the number of attributes to output: %d",
+ tupDesc->natts, list_length(cstate->attnumlist)));
+}
+
+static void
+TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid)));
+}
+
+static void
+TestCopyToEnd(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToEnd")));
+}
+
+static const CopyToRoutine TestCopyToRoutine = {
+ .CopyToOutFunc = TestCopyToOutFunc,
+ .CopyToStart = TestCopyToStart,
+ .CopyToOneRow = TestCopyToOneRow,
+ .CopyToEnd = TestCopyToEnd,
+};
+
+
+static const CopyFromRoutine TestCopyFromRoutine = {
+ .CopyFromInFunc = TestCopyFromInFunc,
+ .CopyFromStart = TestCopyFromStart,
+ .CopyFromOneRow = TestCopyFromOneRow,
+ .CopyFromEnd = TestCopyFromEnd,
+};
+
+void
+_PG_init(void)
+{
+ RegisterCopyCustomFormat("test_format",
+ &TestCopyToRoutine,
+ &TestCopyFromRoutine,
+ &TestCopyProcessOneOption,
+ &TestCopyValidateOptions);
+}
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 5263710e451..552669abc5f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3179,6 +3179,7 @@ Tcl_Obj
Tcl_Size
Tcl_Time
TempNamespaceStatus
+TestCopyOptions
TestDSMRegistryHashEntry
TestDSMRegistryStruct
TestDecodingData
--
2.54.0
[text/x-patch] v3-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch (26.4K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/5-v3-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch)
download | inline diff:
From 47aa927d65e483331f9b2010499b4c7f0bfb562c Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <[email protected]>
Date: Mon, 22 Jun 2026 11:50:27 -0700
Subject: [PATCH v3 1/4] Move Copy[From|To]StateData to copy_state.h.
Author:
Reviewed-by:
Discussion: https://postgr.es/m/
Backpatch-through:
---
contrib/file_fdw/file_fdw.c | 2 +-
src/backend/commands/copyfrom.c | 5 +-
src/backend/commands/copyfromparse.c | 10 +-
src/backend/commands/copyto.c | 88 +-------
src/include/commands/copy.h | 2 +-
src/include/commands/copy_state.h | 253 +++++++++++++++++++++++
src/include/commands/copyfrom_internal.h | 165 +--------------
7 files changed, 273 insertions(+), 252 deletions(-)
create mode 100644 src/include/commands/copy_state.h
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 33a37d832ce..d152d05b92e 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -22,7 +22,7 @@
#include "catalog/pg_authid.h"
#include "catalog/pg_foreign_table.h"
#include "commands/copy.h"
-#include "commands/copyfrom_internal.h"
+#include "commands/copy_state.h"
#include "commands/defrem.h"
#include "commands/explain_format.h"
#include "commands/explain_state.h"
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 0087585b2c4..2c57b32f4de 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -30,6 +30,7 @@
#include "access/xact.h"
#include "catalog/namespace.h"
#include "commands/copyapi.h"
+#include "commands/copy_state.h"
#include "commands/copyfrom_internal.h"
#include "commands/progress.h"
#include "commands/trigger.h"
@@ -1732,7 +1733,7 @@ BeginCopyFrom(ParseState *pstate,
pg_encoding_to_char(GetDatabaseEncoding()))));
}
- cstate->copy_src = COPY_FILE; /* default */
+ cstate->copy_src = COPY_SOURCE_FILE; /* default */
cstate->whereClause = whereClause;
@@ -1861,7 +1862,7 @@ BeginCopyFrom(ParseState *pstate,
if (data_source_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_src = COPY_CALLBACK;
+ cstate->copy_src = COPY_SOURCE_CALLBACK;
cstate->data_source_cb = data_source_cb;
}
else if (pipe)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 65fd5a0ab4f..0ff5db4b62d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -184,7 +184,7 @@ ReceiveCopyBegin(CopyFromState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_src = COPY_FRONTEND;
+ cstate->copy_src = COPY_SOURCE_FRONTEND;
cstate->fe_msgbuf = makeStringInfo();
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -252,7 +252,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
switch (cstate->copy_src)
{
- case COPY_FILE:
+ case COPY_SOURCE_FILE:
pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ);
bytesread = fread(databuf, 1, maxread, cstate->copy_file);
pgstat_report_wait_end();
@@ -263,7 +263,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
if (bytesread == 0)
cstate->raw_reached_eof = true;
break;
- case COPY_FRONTEND:
+ case COPY_SOURCE_FRONTEND:
while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
{
int avail;
@@ -346,7 +346,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
bytesread += avail;
}
break;
- case COPY_CALLBACK:
+ case COPY_SOURCE_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
break;
}
@@ -1259,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (cstate->copy_src == COPY_SOURCE_FRONTEND)
{
int inbytes;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6755bb698de..ef2038c9a5d 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -23,8 +23,8 @@
#include "access/tupconvert.h"
#include "catalog/pg_inherits.h"
#include "commands/copyapi.h"
+#include "commands/copy_state.h"
#include "commands/progress.h"
-#include "executor/execdesc.h"
#include "executor/executor.h"
#include "executor/tuptable.h"
#include "funcapi.h"
@@ -42,76 +42,6 @@
#include "utils/snapmgr.h"
#include "utils/wait_event.h"
-/*
- * Represents the different dest cases we need to worry about at
- * the bottom level
- */
-typedef enum CopyDest
-{
- COPY_FILE, /* to file (or a piped program) */
- COPY_FRONTEND, /* to frontend */
- COPY_CALLBACK, /* to callback function */
-} CopyDest;
-
-/*
- * This struct contains all the state variables used throughout a COPY TO
- * operation.
- *
- * Multi-byte encodings: all supported client-side encodings encode multi-byte
- * characters by having the first byte's high bit set. Subsequent bytes of the
- * character can have the high bit not set. When scanning data in such an
- * encoding to look for a match to a single-byte (ie ASCII) character, we must
- * use the full pg_encoding_mblen() machinery to skip over multibyte
- * characters, else we might find a false match to a trailing byte. In
- * supported server encodings, there is no possibility of a false match, and
- * it's faster to make useless comparisons to trailing bytes than it is to
- * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
- * when we have to do it the hard way.
- */
-typedef struct CopyToStateData
-{
- /* format-specific routines */
- const CopyToRoutine *routine;
-
- /* low-level state data */
- CopyDest copy_dest; /* type of copy source/destination */
- FILE *copy_file; /* used if copy_dest == COPY_FILE */
- StringInfo fe_msgbuf; /* used for all dests during COPY TO */
-
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- bool json_row_delim_needed; /* need delimiter before next row */
- StringInfo json_buf; /* reusable buffer for JSON output,
- * initialized in BeginCopyTo */
- TupleDesc tupDesc; /* Descriptor for JSON output; for a column
- * list this is a projected descriptor */
- Datum *json_projvalues; /* pre-allocated projection values, or
- * NULL */
- bool *json_projnulls; /* pre-allocated projection nulls, or NULL */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
- List *partitions; /* OID list of partitions to copy data from */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- FmgrInfo *out_functions; /* lookup info for output functions */
- MemoryContext rowcontext; /* per-row evaluation context */
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
-
/* DestReceiver for COPY (query) TO */
typedef struct
{
@@ -559,7 +489,7 @@ SendCopyBegin(CopyToState cstate)
}
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->copy_dest = COPY_DEST_FRONTEND;
}
static void
@@ -606,7 +536,7 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
pgstat_report_wait_start(WAIT_EVENT_COPY_TO_WRITE);
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
@@ -642,11 +572,11 @@ CopySendEndOfRow(CopyToState cstate)
}
pgstat_report_wait_end();
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
- case COPY_CALLBACK:
+ case COPY_DEST_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
break;
}
@@ -667,7 +597,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
{
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
/* Default line termination depends on platform */
#ifndef WIN32
CopySendChar(cstate, '\n');
@@ -675,7 +605,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate)
CopySendString(cstate, "\r\n");
#endif
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* The FE/BE protocol uses \n as newline for all platforms */
CopySendChar(cstate, '\n');
break;
@@ -1135,12 +1065,12 @@ BeginCopyTo(ParseState *pstate,
/* See Multibyte encoding comment above */
cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->copy_dest = COPY_DEST_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->copy_dest = COPY_DEST_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index abecfe51098..5e710efff5b 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -99,7 +99,7 @@ typedef struct CopyFormatOptions
List *convert_select; /* list of column names (can be NIL) */
} CopyFormatOptions;
-/* These are private in commands/copy[from|to].c */
+/* These are defined in copy_state.h */
typedef struct CopyFromStateData *CopyFromState;
typedef struct CopyToStateData *CopyToState;
diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h
new file mode 100644
index 00000000000..52cbf5067eb
--- /dev/null
+++ b/src/include/commands/copy_state.h
@@ -0,0 +1,253 @@
+/*-------------------------------------------------------------------------
+ *
+ * copy_state.h
+ * prototypes for COPY TO/COPY FROM execution state.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994-5, Regents of the University of California
+ *
+ * src/include/commands/copy_state.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef COPY_STATE_H
+#define COPY_STATE_H
+
+#include "commands/copy.h"
+#include "commands/trigger.h"
+#include "executor/execdesc.h"
+#include "nodes/miscnodes.h"
+
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_SOURCE_FILE, /* from file (or a piped program) */
+ COPY_SOURCE_FRONTEND, /* from frontend */
+ COPY_SOURCE_CALLBACK, /* from callback function */
+} CopySource;
+
+/*
+ * Represents the end-of-line terminator type of the input
+ */
+typedef enum EolType
+{
+ EOL_UNKNOWN,
+ EOL_NL,
+ EOL_CR,
+ EOL_CRNL,
+} EolType;
+
+/*
+ * This struct contains all the state variables used throughout a COPY FROM
+ * operation.
+ */
+typedef struct CopyFromStateData
+{
+ /* format routine */
+ const struct CopyFromRoutine *routine;
+
+ /* low-level state data */
+ CopySource copy_src; /* type of copy source */
+ FILE *copy_file; /* used if copy_src == COPY_SOURCE_FILE */
+ StringInfo fe_msgbuf; /* used if copy_src == COPY_SOURCE_FRONTEND */
+
+ EolType eol_type; /* EOL type of input */
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ Oid conversion_proc; /* encoding conversion function */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ CopyFormatOptions opts;
+ bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /* these are just for error messages, see CopyFromErrorCallback */
+ const char *cur_relname; /* table name for error messages */
+ uint64 cur_lineno; /* line number for error messages */
+ const char *cur_attname; /* current att for error messages */
+ const char *cur_attval; /* current att value for error messages */
+ bool relname_only; /* don't output line number, att, etc. */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ AttrNumber num_defaults; /* count of att that are missing and have
+ * default value */
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+ ErrorSaveContext *escontext; /* soft error trapped during in_functions
+ * execution */
+ uint64 num_errors; /* total number of rows which contained soft
+ * errors */
+ int *defmap; /* array of default att numbers related to
+ * missing att */
+ ExprState **defexprs; /* array of default att expressions for all
+ * att */
+ bool *defaults; /* if DEFAULT marker was found for
+ * corresponding att */
+ bool simd_enabled; /* use SIMD to scan for special chars? */
+
+ /*
+ * True if the corresponding attribute's is a constrained domain. This
+ * will be populated only when ON_ERROR is SET_NULL, otherwise NULL.
+ */
+ bool *domain_with_constraint;
+
+ bool volatile_defexprs; /* is any of defexprs volatile? */
+ List *range_table; /* single element list of RangeTblEntry */
+ List *rteperminfos; /* single element list of RTEPermissionInfo */
+ ExprState *qualexpr;
+
+ TransitionCaptureState *transition_capture;
+
+ /*
+ * These variables are used to reduce overhead in COPY FROM.
+ *
+ * attribute_buf holds the separated, de-escaped text for each field of
+ * the current line. The CopyReadAttributes functions return arrays of
+ * pointers into this buffer. We avoid palloc/pfree overhead by re-using
+ * the buffer on each cycle.
+ *
+ * In binary COPY FROM, attribute_buf holds the binary data for the
+ * current field, but the usage is otherwise similar.
+ */
+ StringInfoData attribute_buf;
+
+ /* field raw data pointers found by COPY FROM */
+
+ int max_fields;
+ char **raw_fields;
+
+ /*
+ * Similarly, line_buf holds the whole input line being processed. The
+ * input cycle is first to read the whole line into line_buf, and then
+ * extract the individual attribute fields into attribute_buf. line_buf
+ * is preserved unmodified so that we can display it in error messages if
+ * appropriate. (In binary mode, line_buf is not used.)
+ */
+ StringInfoData line_buf;
+ bool line_buf_valid; /* contains the row being processed? */
+
+ /*
+ * input_buf holds input data, already converted to database encoding.
+ *
+ * In text mode, CopyReadLine parses this data sufficiently to locate line
+ * boundaries, then transfers the data to line_buf. We guarantee that
+ * there is a \0 at input_buf[input_buf_len] at all times. (In binary
+ * mode, input_buf is not used.)
+ *
+ * If encoding conversion is not required, input_buf is not a separate
+ * buffer but points directly to raw_buf. In that case, input_buf_len
+ * tracks the number of bytes that have been verified as valid in the
+ * database encoding, and raw_buf_len is the total number of bytes stored
+ * in the buffer.
+ */
+#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
+ char *input_buf;
+ int input_buf_index; /* next byte to process */
+ int input_buf_len; /* total # of bytes stored */
+ bool input_reached_eof; /* true if we reached EOF */
+ bool input_reached_error; /* true if a conversion error happened */
+ /* Shorthand for number of unconsumed bytes available in input_buf */
+#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
+
+ /*
+ * raw_buf holds raw input data read from the data source (file or client
+ * connection), not yet converted to the database encoding. Like with
+ * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
+ */
+#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
+ char *raw_buf;
+ int raw_buf_index; /* next byte to process */
+ int raw_buf_len; /* total # of bytes stored */
+ bool raw_reached_eof; /* true if we reached EOF */
+
+ /* Shorthand for number of unconsumed bytes available in raw_buf */
+#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
+
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyFromStateData;
+
+/*
+ * Represents the different dest cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopyDest
+{
+ COPY_DEST_FILE, /* to file (or a piped program) */
+ COPY_DEST_FRONTEND, /* to frontend */
+ COPY_DEST_CALLBACK, /* to callback function */
+} CopyDest;
+
+/*
+ * This struct contains all the state variables used throughout a COPY TO
+ * operation.
+ *
+ * Multi-byte encodings: all supported client-side encodings encode multi-byte
+ * characters by having the first byte's high bit set. Subsequent bytes of the
+ * character can have the high bit not set. When scanning data in such an
+ * encoding to look for a match to a single-byte (ie ASCII) character, we must
+ * use the full pg_encoding_mblen() machinery to skip over multibyte
+ * characters, else we might find a false match to a trailing byte. In
+ * supported server encodings, there is no possibility of a false match, and
+ * it's faster to make useless comparisons to trailing bytes than it is to
+ * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
+ * when we have to do it the hard way.
+ */
+typedef struct CopyToStateData
+{
+ /* format-specific routines */
+ const struct CopyToRoutine *routine;
+
+ /* low-level state data */
+ CopyDest copy_dest; /* type of copy source/destination */
+ FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */
+ StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ bool json_row_delim_needed; /* need delimiter before next row */
+ StringInfo json_buf; /* reusable buffer for JSON output,
+ * initialized in BeginCopyTo */
+ TupleDesc tupDesc; /* Descriptor for JSON output; for a column
+ * list this is a projected descriptor */
+ Datum *json_projvalues; /* pre-allocated projection values, or
+ * NULL */
+ bool *json_projnulls; /* pre-allocated projection nulls, or NULL */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+ List *partitions; /* OID list of partitions to copy data from */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+ MemoryContext rowcontext; /* per-row evaluation context */
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyToStateData;
+
+#endif /* COPY_STATE_H */
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 9d3e244ee55..f7afade9a39 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -14,31 +14,7 @@
#ifndef COPYFROM_INTERNAL_H
#define COPYFROM_INTERNAL_H
-#include "commands/copy.h"
-#include "commands/trigger.h"
-#include "nodes/miscnodes.h"
-
-/*
- * Represents the different source cases we need to worry about at
- * the bottom level
- */
-typedef enum CopySource
-{
- COPY_FILE, /* from file (or a piped program) */
- COPY_FRONTEND, /* from frontend */
- COPY_CALLBACK, /* from callback function */
-} CopySource;
-
-/*
- * Represents the end-of-line terminator type of the input
- */
-typedef enum EolType
-{
- EOL_UNKNOWN,
- EOL_NL,
- EOL_CR,
- EOL_CRNL,
-} EolType;
+#include "commands/copy_state.h"
/*
* Represents the insert method to be used during COPY FROM.
@@ -52,145 +28,6 @@ typedef enum CopyInsertMethod
* ExecForeignBatchInsert only if valid */
} CopyInsertMethod;
-/*
- * This struct contains all the state variables used throughout a COPY FROM
- * operation.
- */
-typedef struct CopyFromStateData
-{
- /* format routine */
- const struct CopyFromRoutine *routine;
-
- /* low-level state data */
- CopySource copy_src; /* type of copy source */
- FILE *copy_file; /* used if copy_src == COPY_FILE */
- StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
-
- EolType eol_type; /* EOL type of input */
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- Oid conversion_proc; /* encoding conversion function */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
- bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /* these are just for error messages, see CopyFromErrorCallback */
- const char *cur_relname; /* table name for error messages */
- uint64 cur_lineno; /* line number for error messages */
- const char *cur_attname; /* current att for error messages */
- const char *cur_attval; /* current att value for error messages */
- bool relname_only; /* don't output line number, att, etc. */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- AttrNumber num_defaults; /* count of att that are missing and have
- * default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
- ErrorSaveContext *escontext; /* soft error trapped during in_functions
- * execution */
- uint64 num_errors; /* total number of rows which contained soft
- * errors */
- int *defmap; /* array of default att numbers related to
- * missing att */
- ExprState **defexprs; /* array of default att expressions for all
- * att */
- bool *defaults; /* if DEFAULT marker was found for
- * corresponding att */
- bool simd_enabled; /* use SIMD to scan for special chars? */
-
- /*
- * True if the corresponding attribute's is a constrained domain. This
- * will be populated only when ON_ERROR is SET_NULL, otherwise NULL.
- */
- bool *domain_with_constraint;
-
- bool volatile_defexprs; /* is any of defexprs volatile? */
- List *range_table; /* single element list of RangeTblEntry */
- List *rteperminfos; /* single element list of RTEPermissionInfo */
- ExprState *qualexpr;
-
- TransitionCaptureState *transition_capture;
-
- /*
- * These variables are used to reduce overhead in COPY FROM.
- *
- * attribute_buf holds the separated, de-escaped text for each field of
- * the current line. The CopyReadAttributes functions return arrays of
- * pointers into this buffer. We avoid palloc/pfree overhead by re-using
- * the buffer on each cycle.
- *
- * In binary COPY FROM, attribute_buf holds the binary data for the
- * current field, but the usage is otherwise similar.
- */
- StringInfoData attribute_buf;
-
- /* field raw data pointers found by COPY FROM */
-
- int max_fields;
- char **raw_fields;
-
- /*
- * Similarly, line_buf holds the whole input line being processed. The
- * input cycle is first to read the whole line into line_buf, and then
- * extract the individual attribute fields into attribute_buf. line_buf
- * is preserved unmodified so that we can display it in error messages if
- * appropriate. (In binary mode, line_buf is not used.)
- */
- StringInfoData line_buf;
- bool line_buf_valid; /* contains the row being processed? */
-
- /*
- * input_buf holds input data, already converted to database encoding.
- *
- * In text mode, CopyReadLine parses this data sufficiently to locate line
- * boundaries, then transfers the data to line_buf. We guarantee that
- * there is a \0 at input_buf[input_buf_len] at all times. (In binary
- * mode, input_buf is not used.)
- *
- * If encoding conversion is not required, input_buf is not a separate
- * buffer but points directly to raw_buf. In that case, input_buf_len
- * tracks the number of bytes that have been verified as valid in the
- * database encoding, and raw_buf_len is the total number of bytes stored
- * in the buffer.
- */
-#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
- char *input_buf;
- int input_buf_index; /* next byte to process */
- int input_buf_len; /* total # of bytes stored */
- bool input_reached_eof; /* true if we reached EOF */
- bool input_reached_error; /* true if a conversion error happened */
- /* Shorthand for number of unconsumed bytes available in input_buf */
-#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
-
- /*
- * raw_buf holds raw input data read from the data source (file or client
- * connection), not yet converted to the database encoding. Like with
- * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
- */
-#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
- char *raw_buf;
- int raw_buf_index; /* next byte to process */
- int raw_buf_len; /* total # of bytes stored */
- bool raw_reached_eof; /* true if we reached EOF */
-
- /* Shorthand for number of unconsumed bytes available in raw_buf */
-#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
-
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
-
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
--
2.54.0
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 05:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2026-06-24 01:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2026-06-26 03:21 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2026-06-26 03:21 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 23 Jun 2026 18:15:10 -0700,
Masahiko Sawada <[email protected]> wrote:
> Thank you for reviewing the patches! I've attached updated patches.
+1
I have only a few minor comments:
0002:
> --- /dev/null
> +++ b/src/backend/commands/copyapi.c
> +void
> +RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to,
> + const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn)
How about using "const CopyCustomFormatEntry *" instead of
"to", "from" and "option_fn"? If we use
CopyCustomFormatEntry here, we don't need change the
signature of this function when we add more items.
> +const CopyCustomFormatEntry *
> +GetCopyCustomFormatRoutines(const char *name)
How about renaming this to "...FormatEntry" from
"...FormatRoutines"?
> --- a/src/include/commands/copy.h
> +++ b/src/include/commands/copy.h
> +#define CopyFormatIsBuiltins(format) ((format) != COPY_FORMAT_CUSTOM)
How about removing the last "s" ("...IsBuiltin") because
this processes only one format?
> + const struct CopyCustomFormatEntry *custom_format_ent;
It may be better that we don't abbreviate "_entry" to "_ent"
here for readability. It seems that we use this abbreviation
only in a few places:
$ git grep '_ent;' src/
src/backend/replication/logical/reorderbuffer.c: ReorderBufferTupleCidEnt *new_ent;
src/backend/utils/cache/catcache.c: CatCInProgress in_progress_ent;
src/backend/utils/cache/catcache.c: catcache_in_progress_stack = &in_progress_ent;
src/backend/utils/cache/catcache.c: CatCInProgress in_progress_ent;
src/backend/utils/cache/catcache.c: catcache_in_progress_stack = &in_progress_ent;
> I'll verify that the new API works well with an experimental custom
> copy format extension.
I think that we need to provide more APIs to read/write data
like we did in v40-0003 to implement a custom copy format
extension:
https://www.postgresql.org/message-id/flat/20250425.214534.1841428689427124725.kou%40clear-code.com
At least https://github.com/kou/pg-copy-arrow needs them.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-15 12:36 ` Andres Freund <[email protected]>
2025-07-17 20:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Andres Freund @ 2025-07-15 12:36 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
On 2025-07-14 03:28:16 +0900, Masahiko Sawada wrote:
> I've reviewed the 0001 and 0002 patches. The API implemented in the
> 0002 patch looks good to me, but I'm concerned about the capsulation
> of copy state data. With the v42 patches, we pass the whole
> CopyToStateData to the extension codes, but most of the fields in
> CopyToStateData are internal working state data that shouldn't be
> exposed to extensions. I think we need to sort out which fields are
> exposed or not. That way, it would be safer and we would be able to
> avoid exposing copyto_internal.h and extensions would not need to
> include copyfrom_internal.h.
>
> I've implemented a draft patch for that idea. In the 0001 patch, I
> moved fields that are related to internal working state from
> CopyToStateData to CopyToExectuionData. COPY routine APIs pass a
> pointer of CopyToStateData but extensions can access only fields
> except for CopyToExectuionData. In the 0002 patch, I've implemented
> the registration API and some related APIs based on your v42 patch.
> I've made similar changes to COPY FROM codes too.
I've not followed the development of this patch - but I continue to be
concerned about the performance impact it has as-is and the amount of COPY
performance improvements it forecloses.
This seems to add yet another layer of indirection to a lot of hot functions
like CopyGetData() etc.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-15 12:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andres Freund <[email protected]>
@ 2025-07-17 20:33 ` Masahiko Sawada <[email protected]>
2025-07-18 09:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-07-17 20:33 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jul 15, 2025 at 5:37 AM Andres Freund <[email protected]> wrote:
>
> Hi,
>
> On 2025-07-14 03:28:16 +0900, Masahiko Sawada wrote:
> > I've reviewed the 0001 and 0002 patches. The API implemented in the
> > 0002 patch looks good to me, but I'm concerned about the capsulation
> > of copy state data. With the v42 patches, we pass the whole
> > CopyToStateData to the extension codes, but most of the fields in
> > CopyToStateData are internal working state data that shouldn't be
> > exposed to extensions. I think we need to sort out which fields are
> > exposed or not. That way, it would be safer and we would be able to
> > avoid exposing copyto_internal.h and extensions would not need to
> > include copyfrom_internal.h.
> >
> > I've implemented a draft patch for that idea. In the 0001 patch, I
> > moved fields that are related to internal working state from
> > CopyToStateData to CopyToExectuionData. COPY routine APIs pass a
> > pointer of CopyToStateData but extensions can access only fields
> > except for CopyToExectuionData. In the 0002 patch, I've implemented
> > the registration API and some related APIs based on your v42 patch.
> > I've made similar changes to COPY FROM codes too.
>
> I've not followed the development of this patch - but I continue to be
> concerned about the performance impact it has as-is and the amount of COPY
> performance improvements it forecloses.
>
> This seems to add yet another layer of indirection to a lot of hot functions
> like CopyGetData() etc.
>
The most refactoring works have been done by commit 7717f6300 and
2e4127b6d with a slight performance gain. At this stage, we're trying
to introduce the registration API so that extensions can provide their
callbacks to the core. Some functions required for I/O such as
CopyGetData() and CopySendEndOfRow() would be exposed but I'm not
going to add additional indirection function call layers.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-15 12:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andres Freund <[email protected]>
2025-07-17 20:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-07-18 09:49 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-07-18 09:49 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoAQkjU=o0nX4y0jtX0BnsrqA04g2ABqrUwjT88YeEWarA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:33:13 -0700,
Masahiko Sawada <[email protected]> wrote:
>> I've not followed the development of this patch - but I continue to be
>> concerned about the performance impact it has as-is and the amount of COPY
>> performance improvements it forecloses.
>>
>> This seems to add yet another layer of indirection to a lot of hot functions
>> like CopyGetData() etc.
>>
>
> The most refactoring works have been done by commit 7717f6300 and
> 2e4127b6d with a slight performance gain. At this stage, we're trying
> to introduce the registration API so that extensions can provide their
> callbacks to the core. Some functions required for I/O such as
> CopyGetData() and CopySendEndOfRow() would be exposed but I'm not
> going to add additional indirection function call layers.
I think Andres is talking about any indirection not only
indirection function call. In this case, "cstate->XXX" ->
"cstate->edata->XXX".
It's also mentioned in my e-mail. I'm not sure whether it
has performance impact but it's better that we benchmark to
confirm whether there is any performance impact or not with
the Copy{From,To}ExecutionData approach.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-29 16:48 ` David G. Johnston <[email protected]>
2025-03-30 02:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 18:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
3 siblings, 2 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-29 16:48 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote:
>
> The attached v39 patch set uses the followings:
>
> 0001: Create copyto_internal.h and change COPY_XXX to
> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
> (Same as 1. in your suggestion)
> 0002: Support custom format for both COPY TO and COPY FROM.
> (Same as 2. in your suggestion)
> 0003: Expose necessary helper functions such as CopySendEndOfRow()
> and add CopyFromSkipErrorRow().
> (3. + 4. in your suggestion)
> 0004: Define handler functions for built-in formats.
> (Not included in your suggestion)
> 0005: Documentation. (WIP)
> (Same as 5. in your suggestion)
>
>
I don't think this module should be responsible for testing the validity of
"qualified names in a string literal" behavior. Having some of the tests
use a schema qualification, and I'd suggest explicit
double-quoting/case-folding, wouldn't hurt just to demonstrate it's
possible, and how extensions should be referenced, but definitely don't
need tests to prove the broken cases are indeed broken. This relies on an
existing API that has its own tests. It is definitely pointlessly
redundant to have 6 tests that only differ from 6 other tests in their use
of a schema qualification.
I prefer keeping 0002 and 0004 separate. In particular, keeping the design
choice of "unqualified internal format names ignore search_path" should
stand out as its own commit.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-03-30 02:31 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Sutou Kouhei @ 2025-03-30 02:31 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAKFQuwYF7VnYcS9dkfvdzt-dgftMB1DV0bjRcNC8-4iYGS+gjw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 29 Mar 2025 09:48:22 -0700,
"David G. Johnston" <[email protected]> wrote:
> I don't think this module should be responsible for testing the validity of
> "qualified names in a string literal" behavior. Having some of the tests
> use a schema qualification, and I'd suggest explicit
> double-quoting/case-folding, wouldn't hurt just to demonstrate it's
> possible, and how extensions should be referenced, but definitely don't
> need tests to prove the broken cases are indeed broken. This relies on an
> existing API that has its own tests. It is definitely pointlessly
> redundant to have 6 tests that only differ from 6 other tests in their use
> of a schema qualification.
You suggest the followings, right?
1. Add tests for "Schema.Name" with mixed cases
2. Remove the following 6 tests in
src/test/modules/test_copy_format/sql/invalid.sql
----
COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_input_type');
COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_input_type');
COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_return_type');
COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_return_type');
COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_return_value');
COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_return_value');
----
because we have the following 6 tests:
----
SET search_path = public,test_schema;
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_input_type');
COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_input_type');
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_return_type');
COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_return_type');
COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_return_value');
COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_return_value');
RESET search_path;
----
3. Remove the following tests because the behavior must be
tested in other places:
----
COPY public.test FROM stdin WITH (FORMAT 'nonexistent');
COPY public.test TO stdout WITH (FORMAT 'nonexistent');
COPY public.test FROM stdin WITH (FORMAT 'invalid.qualified.name');
COPY public.test TO stdout WITH (FORMAT 'invalid.qualified.name');
----
Does it miss something?
1.: There is no difference between single-quoting and
double-quoting here. Because the information what quote
was used for the given FORMAT value isn't remained
here. Should we update gram.y?
2.: I don't have a strong opinion for it. If nobody objects
it, I'll remove them.
3.: I don't have a strong opinion for it. If nobody objects
it, I'll remove them.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-03-31 18:51 ` Masahiko Sawada <[email protected]>
2025-03-31 20:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-31 18:51 UTC (permalink / raw)
To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Sat, Mar 29, 2025 at 9:49 AM David G. Johnston
<[email protected]> wrote:
>
> On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote:
>>
>>
>> The attached v39 patch set uses the followings:
>>
>> 0001: Create copyto_internal.h and change COPY_XXX to
>> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
>> (Same as 1. in your suggestion)
>> 0002: Support custom format for both COPY TO and COPY FROM.
>> (Same as 2. in your suggestion)
>> 0003: Expose necessary helper functions such as CopySendEndOfRow()
>> and add CopyFromSkipErrorRow().
>> (3. + 4. in your suggestion)
>> 0004: Define handler functions for built-in formats.
>> (Not included in your suggestion)
>> 0005: Documentation. (WIP)
>> (Same as 5. in your suggestion)
>>
>
> I prefer keeping 0002 and 0004 separate. In particular, keeping the design choice of "unqualified internal format names ignore search_path" should stand out as its own commit.
What is the point of having separate commits for already-agreed design
choices? I guess that it would make it easier to revert that decision.
But I think it makes more sense that if we agree with "unqualified
internal format names ignore search_path" the original commit includes
that decision and describes it in the commit message. If we want to
change that design based on the discussion later on, we can have a
separate commit that makes that change and has the link to the
discussion.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-31 18:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-31 20:42 ` David G. Johnston <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-31 20:42 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Mar 31, 2025 at 11:52 AM Masahiko Sawada <[email protected]>
wrote:
> On Sat, Mar 29, 2025 at 9:49 AM David G. Johnston
> <[email protected]> wrote:
> >
> > On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote:
> >>
> >>
> >> The attached v39 patch set uses the followings:
> >>
> >> 0001: Create copyto_internal.h and change COPY_XXX to
> >> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly.
> >> (Same as 1. in your suggestion)
> >> 0002: Support custom format for both COPY TO and COPY FROM.
> >> (Same as 2. in your suggestion)
> >> 0003: Expose necessary helper functions such as CopySendEndOfRow()
> >> and add CopyFromSkipErrorRow().
> >> (3. + 4. in your suggestion)
> >> 0004: Define handler functions for built-in formats.
> >> (Not included in your suggestion)
> >> 0005: Documentation. (WIP)
> >> (Same as 5. in your suggestion)
> >>
> >
> > I prefer keeping 0002 and 0004 separate. In particular, keeping the
> design choice of "unqualified internal format names ignore search_path"
> should stand out as its own commit.
>
> What is the point of having separate commits for already-agreed design
> choices? I guess that it would make it easier to revert that decision.
> But I think it makes more sense that if we agree with "unqualified
> internal format names ignore search_path" the original commit includes
> that decision and describes it in the commit message. If we want to
> change that design based on the discussion later on, we can have a
> separate commit that makes that change and has the link to the
> discussion.
>
Fair. Comment withdrawn. Though I was referring to the WIP patches; I
figured the final patch would squash this all together in any case.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-31 17:05 ` David G. Johnston <[email protected]>
3 siblings, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-31 17:05 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote:
>
> > ---
> > +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
> > + .type = T_CopyFromRoutine,
> > + .CopyFromInFunc = CopyFromInFunc,
> > + .CopyFromStart = CopyFromStart,
> > + .CopyFromOneRow = CopyFromOneRow,
> > + .CopyFromEnd = CopyFromEnd,
> > +};
> >
>
>
In trying to document the current API I'm strongly disliking it. Namely,
the amount of internal code an extension needs to care about/copy-paste to
create a working handler.
Presently, pg_type defines text and binary I/O routines and the text/csv
formats use the text I/O while binary uses binary I/O - for all
attributes. The CopyFromInFunc API allows for each attribute to somehow
have its I/O format individualized. But I don't see how that is practical
or useful, and it adds burden on API users.
I suggest we remove both .CopyFromInFunc and .CopyFromStart/End and add a
property to CopyFromRoutine (.ioMode?) with values of either Copy_IO_Text
or Copy_IO_Binary and then just branch to either:
CopyFromTextLikeInFunc & CopyFromTextLikeStart/End
or
CopyFromBinaryInFunc & CopyFromStart/End
So, in effect, the only method an extension needs to write is converting
to/from the 'serialized' form to the text/binary form (text being near
unanimous).
In a similar manner, the amount of boilerplate within CopyFromOneRow seems
undesirable from an API perspective.
cstate->cur_attname = NameStr(att->attname);
cstate->cur_attval = string;
if (string != NULL)
nulls[m] = false;
if (cstate->defaults[m])
{
/* We must have switched into the per-tuple memory context */
Assert(econtext != NULL);
Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
}
/*
* If ON_ERROR is specified with IGNORE, skip rows with soft errors
*/
else if (!InputFunctionCallSafe(&in_functions[m],
string,
typioparams[m],
att->atttypmod,
(Node *) cstate->escontext,
&values[m]))
{
CopyFromSkipErrorRow(cstate);
return true;
}
cstate->cur_attname = NULL;
cstate->cur_attval = NULL;
It seems to me that CopyFromOneRow could simply produce a *string
collection,
one cell per attribute, and NextCopyFrom could do all of the above on a
for-loop over *string
The pg_type and pg_proc catalogs are not extensible so the API can and
should be limited to
producing the, usually text, values that are ready to be passed into the
text I/O routines and all
the work to find and use types and functions left in the template code.
I haven't looked at COPY TO but I am expecting much the same. The API
should simply receive
the content of the type I/O output routine (binary or text as it dictates)
for each output attribute, by
row, and be expected to take those values and produce a final output from
them.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-04-06 16:20 ` David G. Johnston <[email protected]>
3 siblings, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-04-06 16:20 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Sun, Apr 6, 2025 at 4:30 AM jian he <[email protected]> wrote:
>
> CREATE FUNCTION test_copy_format(internal)
> RETURNS copy_handler
> AS 'MODULE_PATHNAME', 'test_copy_format'
> LANGUAGE C;
> src/backend/commands/copy.c: ProcessCopyOptions
> if (strcmp(fmt, "text") == 0)
> /* default format */ ;
> else if (strcmp(fmt, "csv") == 0)
> opts_out->csv_mode = true;
> else if (strcmp(fmt, "binary") == 0)
> opts_out->binary = true;
> else
> {
> List *qualified_format;
> ....
> }
> what if our customized format name is one of "csv", "binary", "text",
> then that ELSE branch will never be reached.
> then our customized format is being shadowed?
>
>
Yes. The user of your extension can specify a schema name to get access to
your conflicting format name choice but all the existing code out there
that relied on text/csv/binary being the built-in options continue to
behave the same no matter the search_path.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-25 00:45 ` Masahiko Sawada <[email protected]>
1 sibling, 0 replies; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-25 00:45 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Mar 19, 2025 at 6:25 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700,
> "David G. Johnston" <[email protected]> wrote:
>
> >> And could someone help (take over if possible) writing a
> >> document for this feature? I'm not good at writing a
> >> document in English... 0009 in the attached v37 patch set
> >> has a draft of it. It's based on existing documents in
> >> doc/src/sgml/ and *.h.
> >>
> >>
> > I haven't touched the innards of the structs aside from changing
> > programlisting to synopsis. And redoing the two section opening paragraphs
> > to better integrate with the content in the chapter opening.
> >
> > The rest I kinda went to town on...
>
> Thanks!!! It's very helpful!!!
>
> I've applied your patch. 0009 is only changed.
FYI I've implemented an extension to add JSON Lines format as a custom
COPY format[1] to check the usability of the COPY format APIs. I think
that the exposed APIs are fairly simple and minimum. I didn't find the
deficiency and excess of exposed APIs for helping extensions but I
find that it would be better to describe what the one-row callback
should do to utilize the abstracted destination. For example, in order
to use CopyToStateFlush() to write out to the destination, extensions
should write the data to cstate->fe_msgbuf. We expose
CopyToStateFlush() but not for any functions to write data there such
as CopySendString(). It was a bit inconvenient to me but I managed to
write the data directly there by #include'ing copyto_internal.h.
Regards,
[1] https://github.com/MasahikoSawada/pg_copy_jsonlines
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2025-03-22 00:31 ` David G. Johnston <[email protected]>
2025-03-22 05:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-03-22 00:31 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote:
> Hi,
>
> In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format
> implementations" on Mon, 17 Mar 2025 13:50:03 -0700,
> Masahiko Sawada <[email protected]> wrote:
>
> > I think that built-in formats also need to have their handler
> > functions. This seems to be a conventional way for customizable
> > features such as tablesample and access methods, and we can simplify
> > this function.
>
> OK. 0008 in the attached v37 patch set does it.
>
>
tl/dr;
We need to exclude from our SQL function search any function that doesn't
declare copy_handler as its return type.
("function text must return type copy_handler" is not an acceptable error
message)
We need to accept identifiers in FORMAT and parse the optional catalog,
schema, and object name portions.
(Restrict our function search to the named schema if provided.)
Detail:
Fun thing...(not sure how much of this is covered above: I do see, but
didn't scour, the security discussion):
-- create some poison
create function public.text(internal) returns boolean language c as
'/home/davidj/gotya/gotya', 'gotit';
CREATE FUNCTION
-- inject it
postgres=# set search_path to public,pg_catalog;
SET
-- watch it die
postgres=# copy (select 1) to stdout (format text);
ERROR: function text must return type copy_handler
LINE 1: copy (select 1) to stdout (format text);
I'm especially concerned about extensions here.
We shouldn't be locating any SQL function that doesn't have a copy_handler
return type. Unfortunately, LookupFuncName seems incapable of doing what
we want here. I suggest we create a new lookup routine where we can
specify the return argument type as a required element. That would cleanly
mitigate the denial-of-service attack/accident vector demonstrated above
(the text returning function should have zero impact on how this feature
behaves). If someone does create a handler SQL function without using
copy_handler return type we'd end up showing "COPY format 'david' not
recognized" - a developer should be able to figure out they didn't put a
correct return type on their handler function and that is why the system
did not register it.
A second concern is simply people wanting to name things the same; or, why
namespaces were invented.
Can we just accept a proper identifier after FORMAT so we can use
schema-qualified names?
(FORMAT "davescopyformat"."david")
We can special case the internal schema-less names and internally force
pg_catalog to avoid them being shadowed.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-03-22 05:07 ` Masahiko Sawada <[email protected]>
2025-03-22 05:23 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: Masahiko Sawada @ 2025-03-22 05:07 UTC (permalink / raw)
To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Mar 21, 2025 at 5:32 PM David G. Johnston
<[email protected]> wrote:
>
> On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote:
>>
>> Hi,
>>
>> In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com>
>> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Mar 2025 13:50:03 -0700,
>> Masahiko Sawada <[email protected]> wrote:
>>
>> > I think that built-in formats also need to have their handler
>> > functions. This seems to be a conventional way for customizable
>> > features such as tablesample and access methods, and we can simplify
>> > this function.
>>
>> OK. 0008 in the attached v37 patch set does it.
>>
>
> tl/dr;
>
> We need to exclude from our SQL function search any function that doesn't declare copy_handler as its return type.
> ("function text must return type copy_handler" is not an acceptable error message)
>
> We need to accept identifiers in FORMAT and parse the optional catalog, schema, and object name portions.
> (Restrict our function search to the named schema if provided.)
>
> Detail:
>
> Fun thing...(not sure how much of this is covered above: I do see, but didn't scour, the security discussion):
>
> -- create some poison
> create function public.text(internal) returns boolean language c as '/home/davidj/gotya/gotya', 'gotit';
> CREATE FUNCTION
>
> -- inject it
> postgres=# set search_path to public,pg_catalog;
> SET
>
> -- watch it die
> postgres=# copy (select 1) to stdout (format text);
> ERROR: function text must return type copy_handler
> LINE 1: copy (select 1) to stdout (format text);
>
> I'm especially concerned about extensions here.
>
> We shouldn't be locating any SQL function that doesn't have a copy_handler return type. Unfortunately, LookupFuncName seems incapable of doing what we want here. I suggest we create a new lookup routine where we can specify the return argument type as a required element. That would cleanly mitigate the denial-of-service attack/accident vector demonstrated above (the text returning function should have zero impact on how this feature behaves). If someone does create a handler SQL function without using copy_handler return type we'd end up showing "COPY format 'david' not recognized" - a developer should be able to figure out they didn't put a correct return type on their handler function and that is why the system did not register it.
Just to be clear, the patch checks the function's return type before calling it:
funcargtypes[0] = INTERNALOID;
handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
funcargtypes, true);
if (!OidIsValid(handlerOid))
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY format \"%s\" not
recognized", format),
parser_errposition(pstate, defel->location)));
/* check that handler has correct return type */
if (get_func_rettype(handlerOid) != COPY_HANDLEROID)
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("function %s must return type %s",
format, "copy_handler"),
parser_errposition(pstate, defel->location)));
So would changing the error message to like "COPY format 'text' not
recognized" untangle your concern?
FYI the same is true for TABLESAMPLE; it invokes a function with the
specified method name and checks the returned Node type:
=# select * from pg_class tablesample text (0);
ERROR: function text must return type tsm_handler
A difference between TABLESAMPLE and COPY format is that the former
accepts a qualified name but the latter doesn't:
=# create extension tsm_system_rows ;
=# create schema s1;
=# create function s1.system_rows(internal) returns void language c as
'tsm_system_rows.so', 'tsm_system_rows_handler';
=# \df *.system_rows
List of functions
Schema | Name | Result data type | Argument data types | Type
--------+-------------+------------------+---------------------+------
public | system_rows | tsm_handler | internal | func
s1 | system_rows | void | internal | func
(2 rows)
postgres(1:1194923)=# select count(*) from pg_class tablesample system_rows(0);
count
-------
0
(1 row)
postgres(1:1194923)=# select count(*) from pg_class tablesample
s1.system_rows(0);
ERROR: function s1.system_rows must return type tsm_handler
> A second concern is simply people wanting to name things the same; or, why namespaces were invented.
Yeah, I think that the custom COPY format should support qualified
names at least.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-22 05:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2025-03-22 05:23 ` David G. Johnston <[email protected]>
2025-03-22 17:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
0 siblings, 1 reply; 125+ messages in thread
From: David G. Johnston @ 2025-03-22 05:23 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Friday, March 21, 2025, Masahiko Sawada <[email protected]> wrote:
> On Fri, Mar 21, 2025 at 5:32 PM David G. Johnston
> <[email protected]> wrote:
> >
> > On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote:
> >>
> >> Hi,
> >>
> >> In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com>
> >> "Re: Make COPY format extendable: Extract COPY TO format
> implementations" on Mon, 17 Mar 2025 13:50:03 -0700,
> >> Masahiko Sawada <[email protected]> wrote:
> >>
> >> > I think that built-in formats also need to have their handler
> >> > functions. This seems to be a conventional way for customizable
> >> > features such as tablesample and access methods, and we can simplify
> >> > this function.
> >>
> >> OK. 0008 in the attached v37 patch set does it.
> >>
> >
> > tl/dr;
> >
> > We need to exclude from our SQL function search any function that
> doesn't declare copy_handler as its return type.
> > ("function text must return type copy_handler" is not an acceptable
> error message)
> >
> > We need to accept identifiers in FORMAT and parse the optional catalog,
> schema, and object name portions.
> > (Restrict our function search to the named schema if provided.)
> >
> > Detail:
> >
> > Fun thing...(not sure how much of this is covered above: I do see, but
> didn't scour, the security discussion):
> >
> > -- create some poison
> > create function public.text(internal) returns boolean language c as
> '/home/davidj/gotya/gotya', 'gotit';
> > CREATE FUNCTION
> >
> > -- inject it
> > postgres=# set search_path to public,pg_catalog;
> > SET
> >
> > -- watch it die
> > postgres=# copy (select 1) to stdout (format text);
> > ERROR: function text must return type copy_handler
> > LINE 1: copy (select 1) to stdout (format text);
> >
> > I'm especially concerned about extensions here.
> >
> > We shouldn't be locating any SQL function that doesn't have a
> copy_handler return type. Unfortunately, LookupFuncName seems incapable of
> doing what we want here. I suggest we create a new lookup routine where we
> can specify the return argument type as a required element. That would
> cleanly mitigate the denial-of-service attack/accident vector demonstrated
> above (the text returning function should have zero impact on how this
> feature behaves). If someone does create a handler SQL function without
> using copy_handler return type we'd end up showing "COPY format 'david' not
> recognized" - a developer should be able to figure out they didn't put a
> correct return type on their handler function and that is why the system
> did not register it.
>
> Just to be clear, the patch checks the function's return type before
> calling it:
>
> funcargtypes[0] = INTERNALOID;
> handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
>
> funcargtypes, true);
> if (!OidIsValid(handlerOid))
> ereport(ERROR,
> (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
> errmsg("COPY format \"%s\" not
> recognized", format),
> parser_errposition(pstate,
> defel->location)));
>
> /* check that handler has correct return type */
> if (get_func_rettype(handlerOid) != COPY_HANDLEROID)
> ereport(ERROR,
> (errcode(ERRCODE_WRONG_OBJECT_TYPE),
> errmsg("function %s must return type %s",
> format, "copy_handler"),
> parser_errposition(pstate,
> defel->location)));
>
> So would changing the error message to like "COPY format 'text' not
> recognized" untangle your concern?
In my example above copy should not fail at all. The text function created
in public that returns Boolean would never be seen and the real one in
pg_catalog would then be found and behave as expected.
>
> FYI the same is true for TABLESAMPLE; it invokes a function with the
> specified method name and checks the returned Node type:
>
> =# select * from pg_class tablesample text (0);
> ERROR: function text must return type tsm_handler
Then this would benefit from the new function I suggest creating since it
apparently has the same, IMO, bug.
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-22 05:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-22 05:23 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
@ 2025-03-22 17:11 ` David G. Johnston <[email protected]>
0 siblings, 0 replies; 125+ messages in thread
From: David G. Johnston @ 2025-03-22 17:11 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers
On Fri, Mar 21, 2025 at 10:23 PM David G. Johnston <
[email protected]> wrote:
> Then this would benefit from the new function I suggest creating since it
> apparently has the same, IMO, bug.
>
>
Concretely like I posted here:
https://www.postgresql.org/message-id/[email protected]...
David J.
^ permalink raw reply [nested|flat] 125+ messages in thread
end of thread, other threads:[~2026-06-26 03:21 UTC | newest]
Thread overview: 125+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-07-22 15:09 [PATCH v2] Check partitions not in use Alvaro Herrera <[email protected]>
2024-01-24 05:49 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-29 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 16:12 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-02 22:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 02:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-09 08:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 04:29 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-11-17 06:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]>
2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-23 05:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2026-06-24 01:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2026-06-26 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-07-15 12:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andres Freund <[email protected]>
2025-07-17 20:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-07-18 09:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-30 02:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2025-03-31 18:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-31 20:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-31 17:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-04-06 16:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-25 00:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-22 05:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2025-03-22 05:23 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
2025-03-22 17:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox