public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v2] Check partitions not in use 127+ messages / 11 participants [nested] [flat]
* [PATCH v2] Check partitions not in use @ 2019-07-22 15:09 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Alvaro Herrera @ 2019-07-22 15:09 UTC (permalink / raw) --- src/backend/commands/tablecmds.c | 56 +++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index f39f993e01..fd17aa4b5a 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -354,6 +354,7 @@ static void ATSimplePermissions(Relation rel, int allowed_targets); static void ATWrongRelkindError(Relation rel, int allowed_targets); static void ATSimpleRecursion(List **wqueue, Relation rel, AlterTableCmd *cmd, bool recurse, LOCKMODE lockmode); +static void ATCheckPartitionsNotInUse(Relation rel, LOCKMODE lockmode); static void ATTypedTableRecursion(List **wqueue, Relation rel, AlterTableCmd *cmd, LOCKMODE lockmode); static List *find_typed_table_dependencies(Oid typeOid, const char *typeName, @@ -3421,8 +3422,7 @@ CheckTableNotInUse(Relation rel, const char *stmt) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), /* translator: first %s is a SQL command, eg ALTER TABLE */ - errmsg("cannot %s \"%s\" because " - "it is being used by active queries in this session", + errmsg("cannot %s \"%s\" because it is being used by active queries in this session", stmt, RelationGetRelationName(rel)))); if (rel->rd_rel->relkind != RELKIND_INDEX && @@ -3431,8 +3431,7 @@ CheckTableNotInUse(Relation rel, const char *stmt) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), /* translator: first %s is a SQL command, eg ALTER TABLE */ - errmsg("cannot %s \"%s\" because " - "it has pending trigger events", + errmsg("cannot %s \"%s\" because it has pending trigger events", stmt, RelationGetRelationName(rel)))); } @@ -3910,16 +3909,19 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, break; case AT_AddIdentity: ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + /* This command never recurses */ pass = AT_PASS_ADD_CONSTR; break; - case AT_DropIdentity: - ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); - pass = AT_PASS_DROP; - break; case AT_SetIdentity: ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + /* This command never recurses */ pass = AT_PASS_COL_ATTRS; break; + case AT_DropIdentity: + ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + /* This command never recurses */ + pass = AT_PASS_DROP; + break; case AT_DropNotNull: /* ALTER COLUMN DROP NOT NULL */ ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); ATPrepDropNotNull(rel, recurse, recursing); @@ -3985,7 +3987,8 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, break; case AT_DropConstraint: /* DROP CONSTRAINT */ ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); - /* Recursion occurs during execution phase */ + ATCheckPartitionsNotInUse(rel, lockmode); + /* Other recursion occurs during execution phase */ /* No command-specific prep needed except saving recurse flag */ if (recurse) cmd->subtype = AT_DropConstraintRecurse; @@ -5224,8 +5227,9 @@ ATSimpleRecursion(List **wqueue, Relation rel, AlterTableCmd *cmd, bool recurse, LOCKMODE lockmode) { /* - * Propagate to children if desired. Only plain tables and foreign tables - * have children, so no need to search for other relkinds. + * Propagate to children if desired. Only plain tables, foreign tables + * and partitioned tables have children, so no need to search for other + * relkinds. */ if (recurse && (rel->rd_rel->relkind == RELKIND_RELATION || @@ -5259,6 +5263,36 @@ ATSimpleRecursion(List **wqueue, Relation rel, } } +/* + * Obtain list of partitions of the given table, locking them all at the given + * lockmode and ensuring that they all pass CheckTableNotInUse. + * + * This function is a no-op if the given relation is not a partitioned table; + * in particular, nothing is done if it's a legacy inheritance parent. + */ +static void +ATCheckPartitionsNotInUse(Relation rel, LOCKMODE lockmode) +{ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + List *inh; + ListCell *cell; + + inh = find_all_inheritors(RelationGetRelid(rel), lockmode, NULL); + /* first element is the parent rel; must ignore it */ + for_each_cell(cell, inh, list_second_cell(inh)) + { + Relation childrel; + + /* find_all_inheritors already got lock */ + childrel = table_open(lfirst_oid(cell), NoLock); + CheckTableNotInUse(childrel, "ALTER TABLE"); + table_close(childrel, NoLock); + } + list_free(inh); + } +} + /* * ATTypedTableRecursion * -- 2.17.1 --uAKRQypu60I7Lcqm-- ^ permalink raw reply [nested|flat] 127+ messages in thread
* [PATCH v20 5/5] introduce restore_library @ 2023-02-16 06:06 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw) --- contrib/basic_archive/Makefile | 2 + contrib/basic_archive/basic_archive.c | 153 +++++++- contrib/basic_archive/meson.build | 5 + contrib/basic_archive/t/001_restore.pl | 44 +++ doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++- doc/src/sgml/backup.sgml | 40 +- doc/src/sgml/basic-archive.sgml | 31 +- doc/src/sgml/config.sgml | 53 ++- doc/src/sgml/high-availability.sgml | 18 +- src/backend/access/transam/xlog.c | 20 +- src/backend/access/transam/xlogarchive.c | 96 ++++- src/backend/access/transam/xlogrecovery.c | 14 +- src/backend/postmaster/checkpointer.c | 54 +++ src/backend/postmaster/startup.c | 44 ++- src/backend/restore/shell_restore.c | 85 ++++- src/backend/utils/misc/guc_tables.c | 11 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/restore/restore_module.h | 143 +++++++ src/include/restore/shell_restore.h | 16 +- src/tools/pgindent/typedefs.list | 2 + 20 files changed, 1104 insertions(+), 84 deletions(-) create mode 100644 contrib/basic_archive/t/001_restore.pl create mode 100644 src/include/restore/restore_module.h diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile index 100ed81f12..e61f7d676f 100644 --- a/contrib/basic_archive/Makefile +++ b/contrib/basic_archive/Makefile @@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c # typical installcheck users do not have (e.g. buildfarm clients). NO_INSTALLCHECK = 1 +TAP_TESTS = 1 + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c index 6b102e9072..e0d8dce364 100644 --- a/contrib/basic_archive/basic_archive.c +++ b/contrib/basic_archive/basic_archive.c @@ -17,6 +17,11 @@ * a file is successfully archived and then the system crashes before * a durable record of the success has been made. * + * This file also demonstrates a basic restore library implementation that + * is roughly equivalent to the following shell command: + * + * cp /path/to/archivedir/%f %p + * * Copyright (c) 2022-2024, PostgreSQL Global Development Group * * IDENTIFICATION @@ -33,6 +38,7 @@ #include "archive/archive_module.h" #include "common/int.h" #include "miscadmin.h" +#include "restore/restore_module.h" #include "storage/copydir.h" #include "storage/fd.h" #include "utils/guc.h" @@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path); static bool check_archive_directory(char **newval, void **extra, GucSource source); static bool compare_files(const char *file1, const char *file2); static void basic_archive_shutdown(ArchiveModuleState *state); +static bool basic_restore_configured(RestoreModuleState *state); +static bool basic_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +static bool basic_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path); +static bool basic_restore_file(const char *file, const char *path); +static bool basic_restore_timeline_history_exists(RestoreModuleState *state, + const char *file, + const char *path); static const ArchiveModuleCallbacks basic_archive_callbacks = { .startup_cb = basic_archive_startup, @@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = { .shutdown_cb = basic_archive_shutdown }; +static const RestoreModuleCallbacks basic_restore_callbacks = { + .startup_cb = NULL, + .restore_wal_segment_configured_cb = basic_restore_configured, + .restore_wal_segment_cb = basic_restore_wal_segment, + .restore_timeline_history_configured_cb = basic_restore_configured, + .restore_timeline_history_cb = basic_restore_timeline_history, + .timeline_history_exists_configured_cb = basic_restore_configured, + .timeline_history_exists_cb = basic_restore_timeline_history_exists, + .archive_cleanup_configured_cb = NULL, + .archive_cleanup_cb = NULL, + .recovery_end_configured_cb = NULL, + .recovery_end_cb = NULL, + .shutdown_cb = NULL +}; + /* * _PG_init * @@ -71,7 +102,7 @@ void _PG_init(void) { DefineCustomStringVariable("basic_archive.archive_directory", - gettext_noop("Archive file destination directory."), + gettext_noop("Archive file source/destination directory."), NULL, &archive_directory, "", @@ -93,6 +124,17 @@ _PG_archive_module_init(void) return &basic_archive_callbacks; } +/* + * _PG_restore_module_init + * + * Returns the module's restore callbacks. + */ +const RestoreModuleCallbacks * +_PG_restore_module_init(void) +{ + return &basic_restore_callbacks; +} + /* * basic_archive_startup * @@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state) pfree(data); state->private_data = NULL; } + +/* + * basic_restore_configured + * + * Checks that archive_directory is not blank. + */ +static bool +basic_restore_configured(RestoreModuleState *state) +{ + return archive_directory != NULL && archive_directory[0] != '\0'; +} + +/* + * basic_restore_wal_segment + * + * Retrieves one archived WAL segment. + */ +static bool +basic_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName) +{ + return basic_restore_file(file, path); +} + +/* + * basic_restore_timeline_history + * + * Retrieves one timeline history file. + */ +static bool +basic_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path) +{ + return basic_restore_file(file, path); +} + +/* + * basic_restore_file + * + * Retrieves one file. + */ +static bool +basic_restore_file(const char *file, const char *path) +{ + char archive_path[MAXPGPATH]; + struct stat st; + + ereport(DEBUG1, + (errmsg("restoring \"%s\" via basic_archive", + file))); + + /* + * If the file doesn't exist, return false to indicate that there are no + * more files to restore. + */ + snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file); + if (stat(archive_path, &st) != 0) + { + int elevel = (errno == ENOENT) ? DEBUG1 : ERROR; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + archive_path))); + return false; + } + + copy_file(archive_path, path); + + ereport(DEBUG1, + (errmsg("restored \"%s\" via basic_archive", + file))); + return true; +} + +/* + * basic_restore_timeline_history_exists + * + * Check whether a timeline history file is present in the archive directory. + */ +static bool +basic_restore_timeline_history_exists(RestoreModuleState *state, + const char *file, const char *path) +{ + char archive_path[MAXPGPATH]; + struct stat st; + + ereport(DEBUG1, + (errmsg("checking existence of \"%s\" via basic_archive", + file))); + + snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file); + if (stat(archive_path, &st) != 0) + { + int elevel = (errno == ENOENT) ? DEBUG1 : ERROR; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + archive_path))); + return false; + } + + ereport(DEBUG1, + (errmsg("verified existence of \"%s\" via basic_archive", + file))); + return true; +} diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build index cc2f62bf36..8e36fa7ed6 100644 --- a/contrib/basic_archive/meson.build +++ b/contrib/basic_archive/meson.build @@ -31,4 +31,9 @@ tests += { # typical installcheck users do not have (e.g. buildfarm clients). 'runningcheck': false, }, + 'tap': { + 'tests': [ + 't/001_restore.pl', + ], + }, } diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl new file mode 100644 index 0000000000..6c01f736cf --- /dev/null +++ b/contrib/basic_archive/t/001_restore.pl @@ -0,0 +1,44 @@ + +# Copyright (c) 2024, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# start a node +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(has_archiving => 1, allows_streaming => 1); +my $archive_dir = $node->archive_dir; +$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +$node->append_conf('postgresql.conf', "archive_command = ''"); +$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'"); +$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$node->start; + +# backup the node +my $backup = 'backup'; +$node->backup($backup); + +# generate some new WAL files +$node->safe_psql('postgres', "CREATE TABLE test (a INT);"); +$node->safe_psql('postgres', "SELECT pg_switch_wal();"); +$node->safe_psql('postgres', "INSERT INTO test VALUES (1);"); + +# shut down the node (this should archive all WAL files) +$node->stop; + +# restore from the backup +my $restore = PostgreSQL::Test::Cluster->new('restore'); +$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0); +$restore->append_conf('postgresql.conf', "restore_command = ''"); +$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'"); +$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$restore->start; + +# ensure post-backup WAL is replayed +my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;"); +is($result, "1", "check restore content"); + +done_testing(); diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml index a52d15082d..15be548a73 100644 --- a/doc/src/sgml/archive-and-restore-modules.sgml +++ b/doc/src/sgml/archive-and-restore-modules.sgml @@ -8,15 +8,17 @@ <para> PostgreSQL provides infrastructure to create custom modules for continuous - archiving (see <xref linkend="continuous-archiving"/>). While archiving via - a shell command (i.e., <xref linkend="guc-archive-command"/>) is much - simpler, a custom archive module will often be considerably more robust and - performant. + archiving and recovery (see <xref linkend="continuous-archiving"/>). While a + shell command (i.e., <xref linkend="guc-archive-command"/>, + <xref linkend="guc-restore-command"/>) is much simpler, a custom module will + often be considerably more robust and performant. </para> <para> The <filename>contrib/basic_archive</filename> module contains a working - example, which demonstrates some useful techniques. + example, which demonstrates some useful techniques. As demonstrated in + <filename>basic_archive</filename> a single module may serve as both an + archive module and a restore module. </para> <sect1 id="archive-modules"> @@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state); </sect3> </sect2> </sect1> + + <sect1 id="restore-modules"> + <title>Restore Modules</title> + <indexterm zone="restore-modules"> + <primary>Restore Modules</primary> + </indexterm> + + <para> + When a custom <xref linkend="guc-restore-library"/> is configured, + PostgreSQL will use the module for restore actions. It is + ultimately up to the module to decide how to accomplish each task, + but some recommendations are listed at + <xref linkend="backup-pitr-recovery"/>. + </para> + + <para> + Restore modules must at least consist of an initialization function + (see <xref linkend="restore-module-init"/>) and the required callbacks (see + <xref linkend="restore-module-callbacks"/>). However, restore modules are + also permitted to do much more (e.g., declare GUCs and register background + workers). + </para> + + <sect2 id="restore-module-init"> + <title>Initialization Functions</title> + <indexterm zone="restore-module-init"> + <primary>_PG_restore_module_init</primary> + </indexterm> + + <para> + An restore library is loaded by dynamically loading a shared library with + the <xref linkend="guc-restore-library"/>'s name as the library base name. + The normal library search path is used to locate the library. To provide + the required restore module callbacks and to indicate that the library is + actually a restore module, it needs to provide a function named + <function>_PG_restore_module_init</function>. The result of the function + must be a pointer to a struct of type + <structname>RestoreModuleCallbacks</structname>, which contains everything + that the core code needs to know how to make use of the restore module. + The return value needs to be of server lifetime, which is typically + achieved by defining it as a <literal>static const</literal> variable in + global scope. + +<programlisting> +typedef struct RestoreModuleCallbacks +{ + RestoreStartupCB startup_cb; + RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb; + RestoreWalSegmentCB restore_wal_segment_cb; + RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb; + RestoreTimelineHistoryCB restore_timeline_history_cb; + TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb; + TimelineHistoryExistsCB timeline_history_exists_cb; + ArchiveCleanupConfiguredCB archive_cleanup_configured_cb; + ArchiveCleanupCB archive_cleanup_cb; + RecoveryEndConfiguredCB recovery_end_configured_cb; + RecoveryEndCB recovery_end_cb; + RestoreShutdownCB shutdown_cb; +} RestoreModuleCallbacks; +typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void); +</programlisting> + + The <function>restore_wal_segment_configured_cb</function>, + <function>restore_wal_segment_cb</function>, + <function>restore_timeline_history_configured_cb</function>, + <function>restore_timeline_history_cb</function>, + <function>timeline_history_exists_configured_cb</function>, and + <function>timeline_history_exists_cb</function> callbacks are required for + archive recovery but optional for streaming replication. The others are + always optional. + </para> + </sect2> + + <sect2 id="restore-module-callbacks"> + <title>Restore Module Callbacks</title> + + <para> + The restore callbacks define the actual behavior of the module. The server + will call them as required to execute restore actions. + </para> + + <sect3 id="restore-module-startup"> + <title>Startup Callback</title> + + <para> + The <function>startup_cb</function> callback is called shortly after the + module is loaded. This callback can be used to perform any additional + initialization required. If the restore module has state, it can use + <structfield>state->private_data</structfield> to store it. + +<programlisting> +typedef void (*RestoreStartupCB) (RestoreModuleState *state); +</programlisting> + </para> + </sect3> + + <sect3 id="restore-module-restore-wal-segment-configured"> + <title>Restore WAL Segment Configured Callback</title> + <para> + The <function>restore_wal_segment_configured_cb</function> callback is + called to determine whether the module is fully configured and ready to + accept requests to retrieve WAL files (e.g., its configuration parameters + are set to valid values). If no + <function>restore_wal_segment_configured_cb</function> is defined, the + server always assumes the module is not configured to retrieve WAL files. + +<programlisting> +typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will retrieve WAL files + by calling the <function>restore_wal_segment_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>restore_wal_segment_cb</function> callback to retrieve WAL + files. + </para> + </sect3> + + <sect3 id="restore-module-restore-wal-segment"> + <title>Restore WAL Segment Callback</title> + <para> + The <function>restore_wal_segment_cb</function> callback is called to + retrieve a single archived segment of the WAL file series for archive + recovery or streaming replication. + +<programlisting> +typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName); +</programlisting> + + This callback must return <literal>true</literal> only if the file was + successfully retrieved. If the file is not available in the archives, the + callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the WAL file to retrieve, while <replaceable>path</replaceable> + contains the destination's relative path (including the file name). + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point. That is the earliest + file that must be kept to allow a restore to be restartable, so this + information can be used to truncate the archive to just the minimum + required to support restarting from the current restore. + <replaceable>lastRestartPointFileName</replaceable> is typically only used + by warm-standby configurations (see <xref linkend="warm-standby"/>). Note + that if multiple standby servers are restoring from the same archive + directory, you will need to ensure that you do not delete WAL files until + they are no longer needed by any of the servers. + </para> + </sect3> + + <sect3 id="restore-module-restore-timeline-history-configured"> + <title>Restore Timeline History Configured Callback</title> + <para> + The <function>restore_timeline_history_configured_cb</function> callback + is called to determine whether the module is fully configured and ready to + accept requests to retrieve timeline history files (e.g., its + configuration parameters are set to valid values). If no + <function>restore_timeline_history_configured_cb</function> is defined, + the server always assumes the module is not configured to retrieve + timeline history files. + +<programlisting> +typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will retrieve timeline + history files by calling the + <function>restore_timeline_history_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>restore_timeline_history_cb</function> callback to retrieve + timeline history files. + </para> + </sect3> + + <sect3 id="restore-module-restore-timeline-history"> + <title>Restore Timeline History Callback</title> + <para> + The <function>restore_timeline_history_cb</function> callback is called to + retrieve a single archived timeline history file for archive recovery or + streaming replication. + +<programlisting> +typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path); +</programlisting> + + This callback must return <literal>true</literal> only if the file was + successfully retrieved. If the file is not available in the archives, the + callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the WAL file to retrieve, while <replaceable>path</replaceable> + contains the destination's relative path (including the file name). + </para> + </sect3> + + <sect3 id="restore-module-timeline-history-exists-configured"> + <title>Timeline History Exists Configured Callback</title> + <para> + The <function>timeline_history_exists_configured_cb</function> callback is + called to determine whether the module is fully configured and ready to + accept requests to determine whether a timeline history file exists in the + archives (e.g., its configuration parameters are set to valid values). If + no <function>timeline_history_exists_configured_cb</function> is defined, + the server always assumes the module is not configured to check whether + certain timeline history files exist. + +<programlisting> +typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will check whether + certain timeline history files are present in the archives by calling the + <function>timeline_history_exists_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>timeline_history_exists_cb</function> callback to check if + timeline history files exist in the archives. + </para> + </sect3> + + <sect3 id="restore-module-timeline-history-exists"> + <title>Timeline History Exists Callback</title> + <para> + The <function>timeline_history_exists_cb</function> callback is called to + check whether a timeline history file is present in the archives. + +<programlisting> +typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path); +</programlisting> + + This callback must return <literal>true</literal> only if the file is + present in the archives. If the file is not available in the archives, + the callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the timeline history file. + </para> + + <para> + Some restore modules might not have a dedicated way to determine whether a + timeline history file exists. For example, + <varname>restore_command</varname> only tells the server how to retrieve + files. In this case, the <function>timeline_history_exists_cb</function> + callback should copy the file to <replaceable>path</replaceable>, which + contains the destination's relative path (including the file name), and + the restore module should set + <structfield>state->timeline_history_exists_cb_copies</structfield> to + <literal>true</literal>. It is recommended to set this flag in the + module's <function>startup_cb</function> callback. This flag tells the + server that it should verify that the file was successfully retrieved + after invoking this callback. + </para> + </sect3> + + <sect3 id="restore-module-archive-cleanup-configured"> + <title>Archive Cleanup Configured Callback</title> + <para> + The <function>archive_cleanup_configured_cb</function> callback is called + to determine whether the module is fully configured and ready to accept + requests to remove old WAL files from the archives (e.g., its + configuration parameters are set to valid values). If no + <function>archive_cleanup_configured_cb</function> is defined, the server + always assumes the module is not configured to remove old WAL files. + +<programlisting> +typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will remove old WAL + files by calling the <function>archive_cleanup_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>archive_cleanup_cb</function> callback to remove old WAL files. + </para> + </sect3> + + <sect3 id="restore-module-archive-cleanup"> + <title>Archive Cleanup Callback</title> + <para> + The <function>archive_cleanup_cb</function> callback is called at every + restart point by the checkpointer process and is intended to provide a + mechanism for cleaning up old archived WAL files that are no longer + needed by the standby server. + +<programlisting> +typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the + name of the file that includes the last valid restart point, like in + <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>. + </para> + </sect3> + + <sect3 id="restore-module-recovery-end-configured"> + <title>Recovery End Configured Callback</title> + <para> + The <function>recovery_end_configured_cb</function> callback is called to + determine whether the module is fully configured and ready to execute its + <function>recovery_end_cb</function> callback once at the end of recovery + (e.g., its configuration parameters are set to valid values). If no + <function>recovery_end_configured_cb</function> is defined, the server + always assumes the module is not configured to execute its + <function>recovery_end_cb</function> callback. + +<programlisting> +typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will execute the + <function>recovery_end_cb</function> callback once at the end of recovery. + If <literal>false</literal> is returned, the server will not use the + <function>recovery_end_cb</function> callback at the end of recovery. + </para> + </sect3> + + <sect3 id="restore-module-recovery-end"> + <title>Recovery End Callback</title> + <para> + The <function>recovery_end_cb</function> callback is called once at the + end of recovery and is intended to provide a mechanism for cleanup + following the end of replication or recovery. + +<programlisting> +typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point, like in + <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>. + </para> + </sect3> + + <sect3 id="restore-module-shutdown"> + <title>Shutdown Callback</title> + <para> + The <function>shutdown_cb</function> callback is called when a process + that has loaded the restore module exits (e.g., after an error) or the + value of <xref linkend="guc-restore-library"/> changes. If no + <function>shutdown_cb</function> is defined, no special action is taken in + these situations. If the restore module has state, this callback should + free it to avoid leaks. + +<programlisting> +typedef void (*RestoreShutdownCB) (RestoreModuleState *state); + </programlisting> + </para> + </sect3> + </sect2> + </sect1> </chapter> diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index b3468eea3c..e1643f92e0 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true); <para> The key part of all this is to set up a recovery configuration that describes how you want to recover and how far the recovery should - run. The one thing that you absolutely must specify is the <varname>restore_command</varname>, + run. The one thing that you absolutely must specify is either + <varname>restore_command</varname> or a <varname>restore_library</varname>, which tells <productname>PostgreSQL</productname> how to retrieve archived - WAL file segments. Like the <varname>archive_command</varname>, this is + WAL file segments. + </para> + + <para> + Like the <varname>archive_library</varname> parameter, + <varname>restore_library</varname> is a shared library. Since such + libraries are written in <literal>C</literal>, creating your own may + require considerably more effort than writing a shell command. However, + restore modules can be more performance than restoring via shell, and they + will have access to many useful server resources. For more information + about creating a <varname>restore_library</varname>, see + <xref linkend="restore-modules"/>. + </para> + + <para> + Like the <varname>archive_command</varname>, + <varname>restore_command</varname> is a shell command string. It can contain <literal>%f</literal>, which is replaced by the name of the desired WAL file, and <literal>%p</literal>, which is replaced by the path name to copy the WAL file to. @@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' </para> <para> - It is important that the command return nonzero exit status on failure. - The command <emphasis>will</emphasis> be called requesting files that are not - present in the archive; it must return nonzero when so asked. This is not - an error condition. An exception is that if the command was terminated by + It is important that the <varname>restore_command</varname> return nonzero + exit status on failure, or, if you are using a + <varname>restore_library</varname>, that the restore callbacks return + <literal>false</literal> on failure. The command or library + <emphasis>will</emphasis> be called requesting files that are not + present in the archive; it must fail when so asked. This is not + an error condition. An exception is that if the + <varname>restore_command</varname> was terminated by a signal (other than <systemitem>SIGTERM</systemitem>, which is used as part of a database server shutdown) or an error by the shell (such as command not found), then recovery will abort and the server will not start - up. + up. Likewise, if the restore callbacks provided by the + <varname>restore_library</varname> emit an <literal>ERROR</literal> or + <literal>FATAL</literal>, recovery will abort and the server won't start. </para> <para> @@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' close as possible given the available WAL segments). Therefore, a normal recovery will end with a <quote>file not found</quote> message, the exact text of the error message depending upon your choice of - <varname>restore_command</varname>. You may also see an error message + <varname>restore_command</varname> or <varname>restore_library</varname>. + You may also see an error message at the start of recovery for a file named something like <filename>00000001.history</filename>. This is also normal and does not indicate a problem in simple recovery situations; see diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml index b4d43ced20..f3a5a502bd 100644 --- a/doc/src/sgml/basic-archive.sgml +++ b/doc/src/sgml/basic-archive.sgml @@ -8,17 +8,20 @@ </indexterm> <para> - <filename>basic_archive</filename> is an example of an archive module. This - module copies completed WAL segment files to the specified directory. This - may not be especially useful, but it can serve as a starting point for - developing your own archive module. For more information about archive - modules, see <xref linkend="archive-modules"/>. + <filename>basic_archive</filename> is an example of an archive and restore + module. This module copies completed WAL segment files to or from the + specified directory. This may not be especially useful, but it can serve as + a starting point for developing your own archive and restore modules. For + more information about archive and restore modules, see\ + <xref linkend="archive-and-restore-modules"/>. </para> <para> - In order to function, this module must be loaded via + For use as an archive module, this module must be loaded via <xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/> - must be enabled. + must be enabled. For use as a restore module, this module must be loaded + via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see + <xref linkend="runtime-config-wal-archive-recovery"/>). </para> <sect2 id="basic-archive-configuration-parameters"> @@ -34,11 +37,12 @@ </term> <listitem> <para> - The directory where the server should copy WAL segment files. This - directory must already exist. The default is an empty string, which - effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/> - is enabled, the server will accumulate WAL segment files in the - expectation that a value will soon be provided. + The directory where the server should copy WAL segment files to or from. + This directory must already exist. The default is an empty string, + which, when used for archiving, effectively halts WAL archival, but if + <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate + WAL segment files in the expectation that a value will soon be provided. + When an empty string is used for recovery, restore will fail. </para> </listitem> </varlistentry> @@ -46,7 +50,7 @@ <para> These parameters must be set in <filename>postgresql.conf</filename>. - Typical usage might be: + Typical usage as an archive module might be: </para> <programlisting> @@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory' <title>Notes</title> <para> + When <filename>basic_archive</filename> is used as an archive module, Server crashes may leave temporary files with the prefix <filename>archtemp</filename> in the archive directory. It is recommended to delete such files before restarting the server after a crash. It is safe to diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 65a6e6c408..9a2570f87f 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3906,7 +3906,8 @@ include_dir 'conf.d' recovery when the end of archived WAL is reached, but will keep trying to continue recovery by connecting to the sending server as specified by the <varname>primary_conninfo</varname> setting and/or by fetching new WAL - segments using <varname>restore_command</varname>. For this mode, the + segments using <varname>restore_command</varname> or + <varname>restore_library</varname>. For this mode, the parameters from this section and <xref linkend="runtime-config-replication-standby"/> are of interest. Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will @@ -3934,7 +3935,8 @@ include_dir 'conf.d' <listitem> <para> The local shell command to execute to retrieve an archived segment of - the WAL file series. This parameter is required for archive recovery, + the WAL file series. Either <varname>restore_command</varname> or + <xref linkend="guc-restore-library"/> is required for archive recovery, but optional for streaming replication. Any <literal>%f</literal> in the string is replaced by the name of the file to retrieve from the archive, @@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>restore_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-restore-library" xreflabel="restore_library"> + <term><varname>restore_library</varname> (<type>string</type>) + <indexterm> + <primary><varname>restore_library</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + The library to use for restore actions, including retrieving archived + files and executing tasks at restartpoints and at recovery end. Either + <xref linkend="guc-restore-command"/> or + <varname>restore_library</varname> is required for archive recovery, + but optional for streaming replication. If this parameter is set to an + empty string (the default), restoring via shell is enabled, and + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname>, and + <varname>recovery_end_command</varname> are used. If both + <varname>restore_library</varname> and any of + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname>, or + <varname>recovery_end_command</varname> are set, an error will be + raised. Otherwise, the specified shared library is used for restoring. + For more information, see <xref linkend="restore-modules"/>. + </para> + + <para> + This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. </para> </listitem> </varlistentry> @@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>archive_cleanup_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> @@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>recovery_end_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index b48209fc2f..a86a9fe93a 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order. <para> In standby mode, the server continuously applies WAL received from the primary server. The standby server can read WAL from a WAL archive - (see <xref linkend="guc-restore-command"/>) or directly from the primary + (see <xref linkend="guc-restore-command"/> and + <xref linkend="guc-restore-library"/>) or directly from the primary over a TCP connection (streaming replication). The standby server will also attempt to restore any WAL found in the standby cluster's <filename>pg_wal</filename> directory. That typically happens after a server @@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order. <para> At startup, the standby begins by restoring all WAL available in the - archive location, calling <varname>restore_command</varname>. Once it + archive location, either by calling <varname>restore_command</varname> or + by executing the <varname>restore_library</varname>'s callbacks. Once it reaches the end of WAL available there and <varname>restore_command</varname> + or <varname>restore_library</varname> fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory. If that fails, and streaming replication has been configured, the standby tries to connect to the primary server and start streaming WAL @@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order. server (see <xref linkend="backup-pitr-recovery"/>). Create a file <link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm> in the standby's cluster data - directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from + directory. Set <xref linkend="guc-restore-command"/> or + <xref linkend="guc-restore-library"/> to copy files from the WAL archive. If you plan to have multiple standby servers for high availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to <literal>latest</literal> (the default), to make the standby server follow the timeline change @@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order. <note> <para> - <xref linkend="guc-restore-command"/> should return immediately + <xref linkend="guc-restore-command"/> and + <xref linkend="guc-restore-library"/> should return immediately if the file does not exist; the server will retry the command again if necessary. </para> @@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order. <para> If you're using a WAL archive, its size can be minimized using the <xref - linkend="guc-archive-cleanup-command"/> parameter to remove files that are no - longer required by the standby server. + linkend="guc-archive-cleanup-command"/> parameter or the + <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove + files that are no longer required by the standby server. The <application>pg_archivecleanup</application> utility is designed specifically to be used with <varname>archive_cleanup_command</varname> in typical single-standby configurations, see <xref linkend="pgarchivecleanup"/>. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e5d12f1d15..34a21c40a5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -83,7 +83,7 @@ #include "replication/snapbuild.h" #include "replication/walreceiver.h" #include "replication/walsender.h" -#include "restore/shell_restore.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog, TimeLineID newTLI) { /* - * Execute the recovery_end_command, if any. + * Execute the recovery-end callback, if any. * - * The command is provided the archive file cutoff point for use during + * The callback is provided the archive file cutoff point for use during * log shipping replication. All files earlier than this point can be * deleted from the archive, though there is no requirement to do so. */ - if (shell_recovery_end_configured()) + if (recovery_end_configured()) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_recovery_end(lastRestartPointFname); + RestoreCallbacks->recovery_end_cb(restore_module_state, + lastRestartPointFname); } /* @@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags) timestamptz_to_str(xtime)) : 0)); /* - * Finally, execute archive_cleanup_command, if any. + * Finally, execute archive-cleanup callback, if any. * - * The command is provided the archive file cutoff point for use during + * The callback is provided the archive file cutoff point for use during * log shipping replication. All files earlier than this point can be * deleted from the archive, though there is no requirement to do so. */ - if (shell_archive_cleanup_configured()) + if (archive_cleanup_configured()) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_archive_cleanup(lastRestartPointFname); + RestoreCallbacks->archive_cleanup_cb(restore_module_state, + lastRestartPointFname); } return true; diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 5756f52124..3a05581495 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -23,14 +23,21 @@ #include "access/xlog_internal.h" #include "access/xlogarchive.h" #include "common/archive.h" +#include "fmgr.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/pgarch.h" #include "postmaster/startup.h" #include "replication/walsender.h" +#include "restore/restore_module.h" #include "restore/shell_restore.h" #include "storage/fd.h" #include "storage/ipc.h" +#include "utils/memutils.h" + +char *restoreLibrary = ""; +const RestoreModuleCallbacks *RestoreCallbacks = NULL; +RestoreModuleState *restore_module_state; /* * Attempt to retrieve the specified file from off-line archival storage. @@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname, switch (archive_type) { case ARCHIVE_TYPE_WAL_SEGMENT: - if (!shell_restore_file_configured()) + if (!restore_wal_segment_configured()) goto not_available; break; case ARCHIVE_TYPE_TIMELINE_HISTORY: - if (!shell_restore_file_configured()) + if (!restore_timeline_history_configured()) goto not_available; break; case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: - if (!shell_restore_file_configured()) + if (!timeline_history_exists_configured()) goto not_available; break; } @@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname, switch (archive_type) { case ARCHIVE_TYPE_WAL_SEGMENT: - ret = shell_restore_wal_segment(xlogfname, xlogpath, - lastRestartPointFname); + ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state, + xlogfname, xlogpath, + lastRestartPointFname); break; case ARCHIVE_TYPE_TIMELINE_HISTORY: - ret = shell_restore_timeline_history(xlogfname, xlogpath); + ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state, + xlogfname, xlogpath); break; case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: - ret = shell_restore_timeline_history(xlogfname, xlogpath); + ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state, + xlogfname, xlogpath); break; } @@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname, if (ret) { + /* + * Some restore functions might not copy the file (e.g., checking + * whether a timeline history file exists), but they can set a flag to + * tell us if they do. We only need to verify file existence if this + * flag is enabled. + */ + if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS && + !restore_module_state->timeline_history_exists_cb_copies) + return true; + /* * command apparently succeeded, but let's make sure the file is * really there now and has the correct size. @@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog) unlink(archiveStatusPath); /* should we complain about failure? */ } + +/* + * Loads all the restore callbacks into our global RestoreCallbacks. The + * caller is responsible for validating the combination of library/command + * parameters that are set (e.g., restore_command and restore_library cannot + * both be set). + */ +void +LoadRestoreCallbacks(void) +{ + RestoreModuleInit init; + + /* + * If the shell command is enabled, use our special initialization + * function. Otherwise, load the library and call its + * _PG_restore_module_init(). + */ + if (restoreLibrary[0] == '\0') + init = shell_restore_init; + else + init = (RestoreModuleInit) + load_external_function(restoreLibrary, "_PG_restore_module_init", + false, NULL); + + if (init == NULL) + ereport(ERROR, + (errmsg("restore modules have to define the symbol " + "_PG_restore_module_init"))); + + RestoreCallbacks = (*init) (); + + /* restore state should be freed before calling this function */ + Assert(restore_module_state == NULL); + restore_module_state = (RestoreModuleState *) + MemoryContextAllocZero(TopMemoryContext, + sizeof(RestoreModuleState)); + + if (RestoreCallbacks->startup_cb != NULL) + RestoreCallbacks->startup_cb(restore_module_state); +} + +/* + * Call the shutdown callback of the loaded restore module, if defined. Also, + * free the restore module state if it was allocated. + * + * Processes that load restore libraries should register this via + * before_shmem_exit(). + */ +void +call_restore_module_shutdown_cb(int code, Datum arg) +{ + if (RestoreCallbacks != NULL && + RestoreCallbacks->shutdown_cb != NULL && + restore_module_state != NULL) + RestoreCallbacks->shutdown_cb(restore_module_state); + + if (restore_module_state != NULL) + { + pfree(restore_module_state); + restore_module_state = NULL; + } +} diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 07213b1897..1bf0d0ea11 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -51,6 +51,7 @@ #include "replication/slot.h" #include "replication/slotsync.h" #include "replication/walreceiver.h" +#include "restore/restore_module.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1117,18 +1118,21 @@ validateRecoveryParameters(void) if (StandbyModeRequested) { if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) && - (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)) + (!restore_wal_segment_configured() || + !restore_timeline_history_configured() || + !timeline_history_exists_configured())) ereport(WARNING, - (errmsg("specified neither primary_conninfo nor restore_command"), + (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"), errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there."))); } else { - if (recoveryRestoreCommand == NULL || - strcmp(recoveryRestoreCommand, "") == 0) + if (!restore_wal_segment_configured() || + !restore_timeline_history_configured() || + !timeline_history_exists_configured()) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("must specify restore_command when standby mode is not enabled"))); + errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled"))); } /* diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 8ef600ae72..c31d0f3bd2 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -46,6 +46,7 @@ #include "postmaster/bgwriter.h" #include "postmaster/interrupt.h" #include "replication/syncrep.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/condition_variable.h" #include "storage/fd.h" @@ -230,6 +231,25 @@ CheckpointerMain(char *startup_data, size_t startup_data_len) ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(checkpointer_context); + /* + * Load restore_library so that we can use its archive-cleanup callback, + * if one is defined. + * + * We also take this opportunity to set up the before_shmem_exit hook for + * the shutdown callback and to check that only one of restore_library, + * archive_cleanup_command is set. Note that we emit a WARNING upon + * detecting misconfigured parameters, and we clear the callbacks so that + * the archive-cleanup functionality is skipped. We judge this + * functionality to not be important enough to require blocking + * checkpoints or shutting down the server when the parameters are + * misconfigured. + */ + before_shmem_exit(call_restore_module_shutdown_cb, 0); + if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command", + WARNING)) + LoadRestoreCallbacks(); + /* * If an exception is encountered, processing resumes here. * @@ -562,6 +582,10 @@ HandleCheckpointerInterrupts(void) if (ConfigReloadPending) { + bool archive_cleanup_settings_changed = false; + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand); + ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); @@ -577,6 +601,36 @@ HandleCheckpointerInterrupts(void) * because of SIGHUP. */ UpdateSharedMemoryConfig(); + + /* + * If the archive-cleanup settings have changed, call the currently + * loaded shutdown callback and clear all the restore callbacks. + * There's presently no good way to unload a library besides + * restarting the process, and there should be little harm in leaving + * it around, so we just leave it loaded. + */ + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0) + { + archive_cleanup_settings_changed = true; + call_restore_module_shutdown_cb(0, (Datum) 0); + RestoreCallbacks = NULL; + } + + /* + * As in CheckpointerMain(), we only emit a WARNING if we detect that + * both restore_library and archive_cleanup_command are set. We do + * this even if the archive-cleanup settings haven't changed to remind + * the user about the misconfiguration. + */ + if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command", + WARNING) && + archive_cleanup_settings_changed) + LoadRestoreCallbacks(); + + pfree(prevRestoreLibrary); + pfree(prevArchiveCleanupCommand); } if (ShutdownRequestPending) { diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index cc665ce259..f8d3d6b3d2 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -26,6 +26,7 @@ #include "miscadmin.h" #include "postmaster/auxprocess.h" #include "postmaster/startup.h" +#include "restore/restore_module.h" #include "storage/ipc.h" #include "storage/pmsignal.h" #include "storage/procsignal.h" @@ -119,13 +120,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS) * Re-read the config file. * * If one of the critical walreceiver options has changed, flag xlog.c - * to restart it. + * to restart it. Also, check for invalid combinations of the command/library + * parameters and reload the restore callbacks if necessary. */ static void StartupRereadConfig(void) { char *conninfo = pstrdup(PrimaryConnInfo); char *slotname = pstrdup(PrimarySlotName); + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevRestoreCommand = pstrdup(recoveryRestoreCommand); + char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand); bool tempSlot = wal_receiver_create_temp_slot; bool conninfoChanged; bool slotnameChanged; @@ -147,6 +152,30 @@ StartupRereadConfig(void) if (conninfoChanged || slotnameChanged || tempSlotChanged) StartupRequestWalReceiverRestart(); + + /* + * If the restore settings have changed, call the currently loaded + * shutdown callback and load the new callbacks. There's presently no + * good way to unload a library besides restarting the process, and there + * should be little harm in leaving it around, so we just leave it loaded. + */ + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command", + ERROR); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command", + ERROR); + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 || + strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0) + { + call_restore_module_shutdown_cb(0, (Datum) 0); + LoadRestoreCallbacks(); + } + + pfree(prevRestoreLibrary); + pfree(prevRestoreCommand); + pfree(prevRecoveryEndCommand); } /* Handle various signals that might be sent to the startup process */ @@ -261,6 +290,19 @@ StartupProcessMain(char *startup_data, size_t startup_data_len) */ sigprocmask(SIG_SETMASK, &UnBlockSig, NULL); + /* + * Check for invalid combinations of the command/library parameters and + * load the callbacks. + */ + before_shmem_exit(call_restore_module_shutdown_cb, 0); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command", + ERROR); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command", + ERROR); + LoadRestoreCallbacks(); + /* * Do what we came for. */ diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c index 42291625c1..be25f04044 100644 --- a/src/backend/restore/shell_restore.c +++ b/src/backend/restore/shell_restore.c @@ -3,7 +3,8 @@ * shell_restore.c * * These restore functions use a user-specified shell command (e.g., the - * restore_command GUC). + * restore_command GUC). It is used as the default, but other modules may + * define their own restore logic. * * Copyright (c) 2024, PostgreSQL Global Development Group * @@ -22,25 +23,76 @@ #include "common/archive.h" #include "common/percentrepl.h" #include "postmaster/startup.h" +#include "restore/restore_module.h" #include "restore/shell_restore.h" #include "storage/ipc.h" #include "utils/wait_event.h" +static void shell_restore_startup(RestoreModuleState *state); +static bool shell_restore_file_configured(RestoreModuleState *state); static bool shell_restore_file(const char *file, const char *path, const char *lastRestartPointFileName); +static bool shell_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +static bool shell_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path); +static bool shell_archive_cleanup_configured(RestoreModuleState *state); +static void shell_archive_cleanup(RestoreModuleState *state, + const char *lastRestartPointFileName); +static bool shell_recovery_end_configured(RestoreModuleState *state); +static void shell_recovery_end(RestoreModuleState *state, + const char *lastRestartPointFileName); static void ExecuteRecoveryCommand(const char *command, const char *commandName, bool failOnSignal, uint32 wait_event_info, const char *lastRestartPointFileName); +static const RestoreModuleCallbacks shell_restore_callbacks = { + .startup_cb = shell_restore_startup, + .restore_wal_segment_configured_cb = shell_restore_file_configured, + .restore_wal_segment_cb = shell_restore_wal_segment, + .restore_timeline_history_configured_cb = shell_restore_file_configured, + .restore_timeline_history_cb = shell_restore_timeline_history, + .timeline_history_exists_configured_cb = shell_restore_file_configured, + .timeline_history_exists_cb = shell_restore_timeline_history, + .archive_cleanup_configured_cb = shell_archive_cleanup_configured, + .archive_cleanup_cb = shell_archive_cleanup, + .recovery_end_configured_cb = shell_recovery_end_configured, + .recovery_end_cb = shell_recovery_end, + .shutdown_cb = NULL +}; + +/* + * shell_restore_init + * + * Returns the callbacks for restoring via shell. + */ +const RestoreModuleCallbacks * +shell_restore_init(void) +{ + return &shell_restore_callbacks; +} + +/* + * Indicates that our timeline_history_exists_cb only attempts to retrieve the + * file, so the caller should verify the file exists afterwards. + */ +static void +shell_restore_startup(RestoreModuleState *state) +{ + state->timeline_history_exists_cb_copies = true; +} + /* * Attempt to execute a shell-based restore command to retrieve a WAL segment. * * Returns true if the command has succeeded, false otherwise. */ -bool -shell_restore_wal_segment(const char *file, const char *path, +static bool +shell_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, const char *lastRestartPointFileName) { return shell_restore_file(file, path, lastRestartPointFileName); @@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path, * * Returns true if the command has succeeded, false otherwise. */ -bool -shell_restore_timeline_history(const char *file, const char *path) +static bool +shell_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path) { char lastRestartPointFname[MAXPGPATH]; @@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path) /* * Check whether restore_command is supplied. */ -bool -shell_restore_file_configured(void) +static bool +shell_restore_file_configured(RestoreModuleState *state) { return recoveryRestoreCommand[0] != '\0'; } @@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path, /* * Check whether archive_cleanup_command is supplied. */ -bool -shell_archive_cleanup_configured(void) +static bool +shell_archive_cleanup_configured(RestoreModuleState *state) { return archiveCleanupCommand[0] != '\0'; } @@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void) /* * Attempt to execute a shell-based archive cleanup command. */ -void -shell_archive_cleanup(const char *lastRestartPointFileName) +static void +shell_archive_cleanup(RestoreModuleState *state, + const char *lastRestartPointFileName) { ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command", false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND, @@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName) /* * Check whether recovery_end_command is supplied. */ -bool -shell_recovery_end_configured(void) +static bool +shell_recovery_end_configured(RestoreModuleState *state) { return recoveryEndCommand[0] != '\0'; } @@ -181,8 +235,9 @@ shell_recovery_end_configured(void) /* * Attempt to execute a shell-based end-of-recovery command. */ -void -shell_recovery_end(const char *lastRestartPointFileName) +static void +shell_recovery_end(RestoreModuleState *state, + const char *lastRestartPointFileName) { ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true, WAIT_EVENT_RECOVERY_END_COMMAND, diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1e71e7db4a..f84724a0ed 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -70,6 +70,7 @@ #include "replication/slot.h" #include "replication/slotsync.h" #include "replication/syncrep.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, + gettext_noop("Sets the library that will be called for restore actions."), + NULL + }, + &restoreLibrary, + "", + NULL, NULL, NULL + }, + { {"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, gettext_noop("Sets the shell command that will be executed at every restart point."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 2244ee52f7..5265857519 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -284,6 +284,7 @@ # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' +#restore_library = '' # library to use for restore actions #archive_cleanup_command = '' # command to execute at every restartpoint #recovery_end_command = '' # command to execute at completion of recovery diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h new file mode 100644 index 0000000000..cc148e855a --- /dev/null +++ b/src/include/restore/restore_module.h @@ -0,0 +1,143 @@ +/*------------------------------------------------------------------------- + * + * restore_module.h + * Exports for restore modules. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * src/include/restore/restore_module.h + * + *------------------------------------------------------------------------- + */ +#ifndef _RESTORE_MODULE_H +#define _RESTORE_MODULE_H + +/* + * The value of the restore_library GUC. + */ +extern PGDLLIMPORT char *restoreLibrary; + +typedef struct RestoreModuleState +{ + /* + * Private data pointer for use by a restore module. This can be used to + * store state for the module that will be passed to each of its + * callbacks. + */ + void *private_data; + + /* + * Indicates whether the callback that checks for timeline existence + * merely copies the file. If set to true, the server will verify that + * the timeline history file exists after the callback returns. + */ + bool timeline_history_exists_cb_copies; +} RestoreModuleState; + +/* + * Restore module callbacks + * + * These callback functions should be defined by restore libraries and returned + * via _PG_restore_module_init(). For more information about the purpose of + * each callback, refer to the restore modules documentation. + */ +typedef void (*RestoreStartupCB) (RestoreModuleState *state); +typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state); +typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state); +typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, + const char *file, const char *path); +typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state); +typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, + const char *file, const char *path); +typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state); +typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, + const char *lastRestartPointFileName); +typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state); +typedef void (*RecoveryEndCB) (RestoreModuleState *state, + const char *lastRestartPointFileName); +typedef void (*RestoreShutdownCB) (RestoreModuleState *state); + +typedef struct RestoreModuleCallbacks +{ + RestoreStartupCB startup_cb; + RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb; + RestoreWalSegmentCB restore_wal_segment_cb; + RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb; + RestoreTimelineHistoryCB restore_timeline_history_cb; + TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb; + TimelineHistoryExistsCB timeline_history_exists_cb; + ArchiveCleanupConfiguredCB archive_cleanup_configured_cb; + ArchiveCleanupCB archive_cleanup_cb; + RecoveryEndConfiguredCB recovery_end_configured_cb; + RecoveryEndCB recovery_end_cb; + RestoreShutdownCB shutdown_cb; +} RestoreModuleCallbacks; + +/* + * Type of the shared library symbol _PG_restore_module_init that is looked up + * when loading a restore library. + */ +typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void); + +extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void); + +extern void LoadRestoreCallbacks(void); +extern void call_restore_module_shutdown_cb(int code, Datum arg); + +extern const RestoreModuleCallbacks *RestoreCallbacks; +extern RestoreModuleState *restore_module_state; + +/* + * The following *_configured() functions are convenient wrappers around the + * *_configured_cb() callback functions with additional defensive NULL checks. + */ + +static inline bool +restore_wal_segment_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->restore_wal_segment_configured_cb != NULL && + RestoreCallbacks->restore_wal_segment_cb != NULL && + RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state); +} + +static inline bool +restore_timeline_history_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->restore_timeline_history_configured_cb != NULL && + RestoreCallbacks->restore_timeline_history_cb != NULL && + RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state); +} + +static inline bool +timeline_history_exists_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->timeline_history_exists_configured_cb != NULL && + RestoreCallbacks->timeline_history_exists_cb != NULL && + RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state); +} + +static inline bool +archive_cleanup_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->archive_cleanup_configured_cb != NULL && + RestoreCallbacks->archive_cleanup_cb != NULL && + RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state); +} + +static inline bool +recovery_end_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->recovery_end_configured_cb != NULL && + RestoreCallbacks->recovery_end_cb != NULL && + RestoreCallbacks->recovery_end_configured_cb(restore_module_state); +} + +#endif /* _RESTORE_MODULE_H */ diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h index 28b5b7bc92..6352623866 100644 --- a/src/include/restore/shell_restore.h +++ b/src/include/restore/shell_restore.h @@ -12,15 +12,13 @@ #ifndef _SHELL_RESTORE_H #define _SHELL_RESTORE_H -extern bool shell_restore_file_configured(void); -extern bool shell_restore_wal_segment(const char *file, const char *path, - const char *lastRestartPointFileName); -extern bool shell_restore_timeline_history(const char *file, const char *path); +#include "restore/restore_module.h" -extern bool shell_archive_cleanup_configured(void); -extern void shell_archive_cleanup(const char *lastRestartPointFileName); - -extern bool shell_recovery_end_configured(void); -extern void shell_recovery_end(const char *lastRestartPointFileName); +/* + * Since the logic for restoring via shell commands is in the core server and + * does not need to be loaded via a shared library, it has a special + * initialization function. + */ +extern const RestoreModuleCallbacks *shell_restore_init(void); #endif /* _SHELL_RESTORE_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a40c59b2e0..db4cc68a88 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2405,6 +2405,8 @@ ResourceOwnerDesc ResourceReleaseCallback ResourceReleaseCallbackItem ResourceReleasePhase +RestoreModuleCallbacks +RestoreModuleState RestoreOptions RestorePass RestrictInfo -- 2.25.1 --3V7upXqbjpZ4EhLz-- ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-15 06:27 Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-15 06:27 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, If there are no more comments for the current design, I'll start implementing this feature with the following approaches for "Discussing" items: > 3.1 Should we use one function(internal) for COPY TO/FROM > handlers or two function(internal)s (one is for COPY TO > handler and another is for COPY FROM handler)? > [4] I'll choose "one function(internal) for COPY TO/FROM handlers". > 3.4 Should we export Copy{To,From}State? Or should we just > provide getters/setters to access Copy{To,From}State > internal? > [10] I'll export Copy{To,From}State. Thanks, -- kou In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > Hi, > > Here is the current summary for a this discussion to make > COPY format extendable. It's for reaching consensus and > starting implementing the feature. (I'll start implementing > the feature once we reach consensus.) If you have any > opinion, please share it. > > Confirmed: > > 1.1 Making COPY format extendable will not reduce performance. > [1] > > Decisions: > > 2.1 Use separated handler for COPY TO and COPY FROM because > our COPY TO implementation (copyto.c) and COPY FROM > implementation (coypfrom.c) are separated. > [2] > > 2.2 Don't use system catalog for COPY TO/FROM handlers. We can > just use a function(internal) that returns a handler instead. > [3] > > 2.3 The implementation must include documentation. > [5] > > 2.4 The implementation must include test. > [6] > > 2.5 The implementation should be consist of small patches > for easy to review. > [6] > > 2.7 Copy{To,From}State must have a opaque space for > handlers. > [8] > > 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO > handlers. > [8] > > 2.9 Make "format" in PgMsg_CopyOutResponse message > extendable. > [9] > > 2.10 Make makeNode() call avoidable in function(internal) > that returns COPY TO/FROM handler. > [9] > > 2.11 Custom COPY TO/FROM handlers must be able to parse > their options. > [11] > > Discussing: > > 3.1 Should we use one function(internal) for COPY TO/FROM > handlers or two function(internal)s (one is for COPY TO > handler and another is for COPY FROM handler)? > [4] > > 3.2 If we use separated function(internal) for COPY TO/FROM > handlers, we need to define naming rule. For example, > <method_name>_to(internal) for COPY TO handler and > <method_name>_from(internal) for COPY FROM handler. > [4] > > 3.3 Should we use prefix or suffix for function(internal) > name to avoid name conflict with other handlers such as > tablesample handlers? > [7] > > 3.4 Should we export Copy{To,From}State? Or should we just > provide getters/setters to access Copy{To,From}State > internal? > [10] > > > [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com > [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz > [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m... > [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m... > [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp... > [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz > [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com > [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com > [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz > [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m... > [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com > > > Thanks, > -- > kou > > ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 05:49 Sutou Kouhei <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Sutou Kouhei @ 2024-01-24 05:49 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, I've implemented custom COPY format feature based on the current design discussion. See the attached patches for details. I also implemented a PoC COPY format handler for Apache Arrow with this implementation and it worked. https://github.com/kou/pg-copy-arrow The patches implement not only custom COPY TO format feature but also custom COPY FROM format feature. 0001-0004 is for COPY TO and 0005-0008 is for COPY FROM. For COPY TO: 0001: This adds CopyToRoutine and use it for text/csv/binary formats. No implementation change. This just move codes. 0002: This adds support for adding custom COPY TO format by "CREATE FUNCTION ${FORMAT_NAME}". This uses the same approach provided by Sawada-san[1] but this doesn't introduce a wrapper CopyRoutine struct for Copy{To,From}Routine. Because I noticed that a wrapper CopyRoutine struct is needless. Copy handler can just return CopyToRoutine or CopyFromRtouine because both of them have NodeTag. We can distinct a returned struct by the NodeTag. [1] https://www.postgresql.org/message-id/[email protected]... 0003: This exports CopyToStateData. No implementation change except CopyDest enum values. I changed COPY_ prefix to COPY_DEST_ to avoid name conflict with CopySource enum values. This just moves codes. 0004: This adds CopyToState::opaque and exports CopySendEndOfRow(). CopySendEndOfRow() is renamed to CopyToStateFlush(). For COPY FROM: 0005: Same as 0001 but for COPY FROM. This adds CopyFromRoutine and use it for text/csv/binary formats. No implementation change. This just move codes. 0006: Same as 0002 but for COPY FROM. This adds support for adding custom COPY FROM format by "CREATE FUNCTION ${FORMAT_NAME}". 0007: Same as 0003 but for COPY FROM. This exports CopyFromStateData. No implementation change except CopySource enum values. I changed COPY_ prefix to COPY_SOURCE_ to align CopyDest changes in 0003. This just moves codes. 0008: Same as 0004 but for COPY FROM. This adds CopyFromState::opaque and exports CopyReadBinaryData(). CopyReadBinaryData() is renamed to CopyFromStateRead(). Thanks, -- kou In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Jan 2024 15:27:02 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > Hi, > > If there are no more comments for the current design, I'll > start implementing this feature with the following > approaches for "Discussing" items: > >> 3.1 Should we use one function(internal) for COPY TO/FROM >> handlers or two function(internal)s (one is for COPY TO >> handler and another is for COPY FROM handler)? >> [4] > > I'll choose "one function(internal) for COPY TO/FROM handlers". > >> 3.4 Should we export Copy{To,From}State? Or should we just >> provide getters/setters to access Copy{To,From}State >> internal? >> [10] > > I'll export Copy{To,From}State. > > > Thanks, > -- > kou > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST), > Sutou Kouhei <[email protected]> wrote: > >> Hi, >> >> Here is the current summary for a this discussion to make >> COPY format extendable. It's for reaching consensus and >> starting implementing the feature. (I'll start implementing >> the feature once we reach consensus.) If you have any >> opinion, please share it. >> >> Confirmed: >> >> 1.1 Making COPY format extendable will not reduce performance. >> [1] >> >> Decisions: >> >> 2.1 Use separated handler for COPY TO and COPY FROM because >> our COPY TO implementation (copyto.c) and COPY FROM >> implementation (coypfrom.c) are separated. >> [2] >> >> 2.2 Don't use system catalog for COPY TO/FROM handlers. We can >> just use a function(internal) that returns a handler instead. >> [3] >> >> 2.3 The implementation must include documentation. >> [5] >> >> 2.4 The implementation must include test. >> [6] >> >> 2.5 The implementation should be consist of small patches >> for easy to review. >> [6] >> >> 2.7 Copy{To,From}State must have a opaque space for >> handlers. >> [8] >> >> 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO >> handlers. >> [8] >> >> 2.9 Make "format" in PgMsg_CopyOutResponse message >> extendable. >> [9] >> >> 2.10 Make makeNode() call avoidable in function(internal) >> that returns COPY TO/FROM handler. >> [9] >> >> 2.11 Custom COPY TO/FROM handlers must be able to parse >> their options. >> [11] >> >> Discussing: >> >> 3.1 Should we use one function(internal) for COPY TO/FROM >> handlers or two function(internal)s (one is for COPY TO >> handler and another is for COPY FROM handler)? >> [4] >> >> 3.2 If we use separated function(internal) for COPY TO/FROM >> handlers, we need to define naming rule. For example, >> <method_name>_to(internal) for COPY TO handler and >> <method_name>_from(internal) for COPY FROM handler. >> [4] >> >> 3.3 Should we use prefix or suffix for function(internal) >> name to avoid name conflict with other handlers such as >> tablesample handlers? >> [7] >> >> 3.4 Should we export Copy{To,From}State? Or should we just >> provide getters/setters to access Copy{To,From}State >> internal? >> [10] >> >> >> [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com >> [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz >> [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m... >> [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m... >> [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp... >> [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz >> [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com >> [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com >> [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz >> [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m... >> [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com >> >> >> Thanks, >> -- >> kou >> >> > > ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 08:11 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-24 08:11 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: > For COPY TO: > > 0001: This adds CopyToRoutine and use it for text/csv/binary > formats. No implementation change. This just move codes. 10M without this change: format,elapsed time (ms) text,1090.763 csv,1136.103 binary,1137.141 10M with this change: format,elapsed time (ms) text,1082.654 csv,1196.991 binary,1069.697 These numbers point out that binary is faster by 6%, csv is slower by 5%, while text stays around what looks like noise range. That's not negligible. Are these numbers reproducible? If they are, that could be a problem for anybody doing bulk-loading of large data sets. I am not sure to understand where the improvement for binary comes from by reading the patch, but perhaps perf would tell more for each format? The loss with csv could be blamed on the extra manipulations of the function pointers, likely. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 12:15 Andrew Dunstan <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Andrew Dunstan @ 2024-01-24 12:15 UTC (permalink / raw) To: Michael Paquier <[email protected]>; Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers On 2024-01-24 We 03:11, Michael Paquier wrote: > On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: >> For COPY TO: >> >> 0001: This adds CopyToRoutine and use it for text/csv/binary >> formats. No implementation change. This just move codes. > 10M without this change: > > format,elapsed time (ms) > text,1090.763 > csv,1136.103 > binary,1137.141 > > 10M with this change: > > format,elapsed time (ms) > text,1082.654 > csv,1196.991 > binary,1069.697 > > These numbers point out that binary is faster by 6%, csv is slower by > 5%, while text stays around what looks like noise range. That's not > negligible. Are these numbers reproducible? If they are, that could > be a problem for anybody doing bulk-loading of large data sets. I am > not sure to understand where the improvement for binary comes from by > reading the patch, but perhaps perf would tell more for each format? > The loss with csv could be blamed on the extra manipulations of the > function pointers, likely. I don't think that's at all acceptable. We've spent quite a lot of blood sweat and tears over the years to make COPY fast, and we should not sacrifice any of that lightly. cheers andrew -- Andrew Dunstan EDB: https://www.enterprisedb.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 14:17 Sutou Kouhei <[email protected]> parent: Andrew Dunstan <[email protected]> 0 siblings, 3 replies; 127+ messages in thread From: Sutou Kouhei @ 2024-01-24 14:17 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, Andrew Dunstan <[email protected]> wrote: > > On 2024-01-24 We 03:11, Michael Paquier wrote: >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: >>> For COPY TO: >>> >>> 0001: This adds CopyToRoutine and use it for text/csv/binary >>> formats. No implementation change. This just move codes. >> 10M without this change: >> >> format,elapsed time (ms) >> text,1090.763 >> csv,1136.103 >> binary,1137.141 >> >> 10M with this change: >> >> format,elapsed time (ms) >> text,1082.654 >> csv,1196.991 >> binary,1069.697 >> >> These numbers point out that binary is faster by 6%, csv is slower by >> 5%, while text stays around what looks like noise range. That's not >> negligible. Are these numbers reproducible? If they are, that could >> be a problem for anybody doing bulk-loading of large data sets. I am >> not sure to understand where the improvement for binary comes from by >> reading the patch, but perhaps perf would tell more for each format? >> The loss with csv could be blamed on the extra manipulations of the >> function pointers, likely. > > > I don't think that's at all acceptable. > > We've spent quite a lot of blood sweat and tears over the years to make COPY > fast, and we should not sacrifice any of that lightly. These numbers aren't reproducible. Because these benchmarks executed on my normal machine not a machine only for benchmarking. The machine runs another processes such as editor and Web browser. For example, here are some results with master (94edfe250c6a200d2067b0debfe00b4122e9b11e): Format,N records,Elapsed time (ms) csv,10000000,1073.715 csv,10000000,1022.830 csv,10000000,1073.584 csv,10000000,1090.651 csv,10000000,1052.259 Here are some results with master + the 0001 patch: Format,N records,Elapsed time (ms) csv,10000000,1025.356 csv,10000000,1067.202 csv,10000000,1014.563 csv,10000000,1032.088 csv,10000000,1058.110 I uploaded my benchmark script so that you can run the same benchmark on your machine: https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 Could anyone try the benchmark with master and master+0001? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 14:20 Sutou Kouhei <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2024-01-24 14:20 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 14:49:36 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > I've implemented custom COPY format feature based on the > current design discussion. See the attached patches for > details. I forgot to mention one note. Documentation isn't included in these patches. I'll write it after all (or some) patches are merged. Is it OK? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 02:53 jian he <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 2 replies; 127+ messages in thread From: jian he @ 2024-01-25 02:53 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 10:17 PM Sutou Kouhei <[email protected]> wrote: > > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 > > Could anyone try the benchmark with master and master+0001? > sorry. I made a mistake. I applied v6, 0001 to 0008 all the patches. my tests: CREATE unlogged TABLE data (a bigint); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, 1e7); my setup: meson setup --reconfigure ${BUILD} \ -Dprefix=${PG_PREFIX} \ -Dpgport=5462 \ -Dbuildtype=release \ -Ddocs_html_style=website \ -Ddocs_pdf=disabled \ -Dllvm=disabled \ -Dextra_version=_release_build gcc version: PostgreSQL 17devel_release_build on x86_64-linux, compiled by gcc-11.4.0, 64-bit apply your patch: COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Time: 668.996 ms Time: 596.254 ms Time: 592.723 ms Time: 591.663 ms Time: 590.803 ms not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Time: 644.246 ms Time: 583.075 ms Time: 568.670 ms Time: 569.463 ms Time: 569.201 ms I forgot to test other formats. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 03:17 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-25 03:17 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 11:17:26PM +0900, Sutou Kouhei wrote: > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, > Andrew Dunstan <[email protected]> wrote: >> We've spent quite a lot of blood sweat and tears over the years to make COPY >> fast, and we should not sacrifice any of that lightly. Clearly. > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 Thanks, that saves time. I am attaching it to this email as well, for the sake of the archives if this link is removed in the future. > Could anyone try the benchmark with master and master+0001? Yep. It is one point we need to settle before deciding what to do with this patch set, and I've done so to reach my own conclusion. I have a rather good machine at my disposal in the cloud, so I did a few runs with HEAD and HEAD+0001, with PGDATA mounted on a tmpfs. Here are some results for the 10M row case, as these should be the least prone to noise, 5 runs each: master text 10M 1732.570 1684.542 1693.430 1687.696 1714.845 csv 10M 1729.113 1724.926 1727.414 1726.237 1728.865 bin 10M 1679.097 1677.887 1676.764 1677.554 1678.120 master+0001 text 10M 1702.207 1654.818 1647.069 1690.568 1654.446 csv 10M 1764.939 1714.313 1712.444 1712.323 1716.952 bin 10M 1703.061 1702.719 1702.234 1703.346 1704.137 Hmm. The point of contention in the patch is the change to use the CopyToOneRow callback in CopyOneRowTo(), as we go through it for each row and we should habe a worst-case scenario with a relation that has a small attribute size. The more rows, the more effect it would have. The memory context switches and the StringInfo manipulations are equally important, and there are a bunch of the latter, actually, with optimizations around fe_msgbuf. I've repeated a few runs across these two builds, and there is some variance and noise, but I am going to agree with your point that the effect 0001 cannot be seen. Even HEAD is showing some noise. So I am discarding the concerns I had after seeing the numbers you posted upthread. +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); +typedef void (*CopyToEnd_function) (CopyToState cstate); We don't really need a set of typedefs here, let's put the definitions in the CopyToRoutine struct instead. +extern CopyToRoutine CopyToRoutineText; +extern CopyToRoutine CopyToRoutineCSV; +extern CopyToRoutine CopyToRoutineBinary; All that should IMO remain in copyto.c and copyfrom.c in the initial patch doing the refactoring. Why not using a fetch function instead that uses a string in input? Then you can call that once after parsing the List of options in ProcessCopyOptions(). Introducing copyapi.h in the initial patch makes sense here for the TO and FROM routines. +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ Some areas, like this comment, are written in an incorrect format. + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, colname); You are right that this is not worth the trouble of creating a different set of callbacks for CSV. This makes the result cleaner. + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); Actually, this split is interesting. It is possible for a custom format to plug in a custom set of out functions. Did you make use of something custom for your own stuff? Actually, could it make sense to split the assignment of cstate->out_functions into its own callback? Sure, that's part of the start phase, but at least it would make clear that a custom method *has* to assign these OIDs to work. The patch implies that as a rule, without a comment that CopyToStart *must* set up these OIDs. I think that 0001 and 0005 should be handled first, as pieces independent of the rest. Then we could move on with 0002~0004 and 0006~0008. -- Michael #!/bin/bash set -eu set -o pipefail base_dir=$(dirname "$0") prepare_sql() { size=$1 db_name=$2 cat <<SQL DROP DATABASE IF EXISTS ${db_name}; CREATE DATABASE ${db_name}; \\c ${db_name} CREATE TABLE data (int32 integer); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${size}); SQL } measure() { local format=$1 local n_tries=6 for i in $(seq ${n_tries}); do LANG=C \ PAGER=cat \ psql \ --no-psqlrc \ --command "\\timing" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \ grep --text -o '^Time: [0-9.]* ms' | \ grep -o '[0-9.]*' done | sort --numeric-sort | head -n 9 | tail -n 1 } result_suffix="" if [ $# -gt 0 ]; then result_suffix="-$1" fi result="${base_dir}/result${result_suffix}.csv" echo "Format,N records,Elapsed time (ms)" | \ tee "${result}" sizes=() sizes+=(100000) sizes+=(1000000) sizes+=(10000000) for size in "${sizes[@]}"; do export PGDATABASE="copy_${size}" echo "${size}: preparing" prepare_sql "${size}" "${PGDATABASE}" | \ psql -d postgres echo "${size}: text" elapsed_time=$(measure text) echo "text,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: csv" elapsed_time=$(measure csv) echo "csv,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: binary" elapsed_time=$(measure binary) echo "binary,${size},${elapsed_time}" | \ tee -a "${result}" done Attachments: [text/plain] bench-run.txt (1.6K, ../../[email protected]/2-bench-run.txt) download | inline: #!/bin/bash set -eu set -o pipefail base_dir=$(dirname "$0") prepare_sql() { size=$1 db_name=$2 cat <<SQL DROP DATABASE IF EXISTS ${db_name}; CREATE DATABASE ${db_name}; \\c ${db_name} CREATE TABLE data (int32 integer); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${size}); SQL } measure() { local format=$1 local n_tries=6 for i in $(seq ${n_tries}); do LANG=C \ PAGER=cat \ psql \ --no-psqlrc \ --command "\\timing" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \ grep --text -o '^Time: [0-9.]* ms' | \ grep -o '[0-9.]*' done | sort --numeric-sort | head -n 9 | tail -n 1 } result_suffix="" if [ $# -gt 0 ]; then result_suffix="-$1" fi result="${base_dir}/result${result_suffix}.csv" echo "Format,N records,Elapsed time (ms)" | \ tee "${result}" sizes=() sizes+=(100000) sizes+=(1000000) sizes+=(10000000) for size in "${sizes[@]}"; do export PGDATABASE="copy_${size}" echo "${size}: preparing" prepare_sql "${size}" "${PGDATABASE}" | \ psql -d postgres echo "${size}: text" elapsed_time=$(measure text) echo "text,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: csv" elapsed_time=$(measure csv) echo "csv,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: binary" elapsed_time=$(measure binary) echo "binary,${size},${elapsed_time}" | \ tee -a "${result}" done [application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 03:28 Michael Paquier <[email protected]> parent: jian he <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Michael Paquier @ 2024-01-25 03:28 UTC (permalink / raw) To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 10:53:58AM +0800, jian he wrote: > apply your patch: > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 > Time: 668.996 ms > Time: 596.254 ms > Time: 592.723 ms > Time: 591.663 ms > Time: 590.803 ms > > not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 > Time: 644.246 ms > Time: 583.075 ms > Time: 568.670 ms > Time: 569.463 ms > Time: 569.201 ms > > I forgot to test other formats. There can be some variance in the tests, so you'd better run much more tests so as you can get a better idea of the mean. Discarding the N highest and lowest values also reduces slightly the effects of the noise you would get across single runs. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 04:36 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 2 replies; 127+ messages in thread From: Masahiko Sawada @ 2024-01-25 04:36 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 11:17 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, > Andrew Dunstan <[email protected]> wrote: > > > > > On 2024-01-24 We 03:11, Michael Paquier wrote: > >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: > >>> For COPY TO: > >>> > >>> 0001: This adds CopyToRoutine and use it for text/csv/binary > >>> formats. No implementation change. This just move codes. > >> 10M without this change: > >> > >> format,elapsed time (ms) > >> text,1090.763 > >> csv,1136.103 > >> binary,1137.141 > >> > >> 10M with this change: > >> > >> format,elapsed time (ms) > >> text,1082.654 > >> csv,1196.991 > >> binary,1069.697 > >> > >> These numbers point out that binary is faster by 6%, csv is slower by > >> 5%, while text stays around what looks like noise range. That's not > >> negligible. Are these numbers reproducible? If they are, that could > >> be a problem for anybody doing bulk-loading of large data sets. I am > >> not sure to understand where the improvement for binary comes from by > >> reading the patch, but perhaps perf would tell more for each format? > >> The loss with csv could be blamed on the extra manipulations of the > >> function pointers, likely. > > > > > > I don't think that's at all acceptable. > > > > We've spent quite a lot of blood sweat and tears over the years to make COPY > > fast, and we should not sacrifice any of that lightly. > > These numbers aren't reproducible. Because these benchmarks > executed on my normal machine not a machine only for > benchmarking. The machine runs another processes such as > editor and Web browser. > > For example, here are some results with master > (94edfe250c6a200d2067b0debfe00b4122e9b11e): > > Format,N records,Elapsed time (ms) > csv,10000000,1073.715 > csv,10000000,1022.830 > csv,10000000,1073.584 > csv,10000000,1090.651 > csv,10000000,1052.259 > > Here are some results with master + the 0001 patch: > > Format,N records,Elapsed time (ms) > csv,10000000,1025.356 > csv,10000000,1067.202 > csv,10000000,1014.563 > csv,10000000,1032.088 > csv,10000000,1058.110 > > > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 > > Could anyone try the benchmark with master and master+0001? > I've run a similar scenario: create unlogged table test (a int); insert into test select c from generate_series(1, 25000000) c; copy test to '/tmp/result.csv' with (format csv); -- generates 230MB file I've run it on HEAD and HEAD+0001 patch and here are the medians of 10 executions for each format: HEAD: binary 2930.353 ms text 2754.852 ms csv 2890.012 ms HEAD w/ 0001 patch: binary 2814.838 ms text 2900.845 ms csv 3015.210 ms Hmm I can see a similar trend that Suto-san had; the binary format got slightly faster whereas both text and csv format has small regression (4%~5%). I think that the improvement for binary came from the fact that we removed "if (cstate->opts.binary)" branches from the original CopyOneRowTo(). I've experimented with a similar optimization for csv and text format; have different callbacks for text and csv format and remove "if (cstate->opts.csv_mode)" branches. I've attached a patch for that. Here are results: HEAD w/ 0001 patch + remove branches: binary 2824.502 ms text 2715.264 ms csv 2803.381 ms The numbers look better now. I'm not sure these are within a noise range but it might be worth considering having different callbacks for text and csv formats. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [application/octet-stream] add_callback_for_csv_format.patch (3.9K, ../../CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com/2-add_callback_for_csv_format.patch) download | inline diff: diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6547b7c654..f18b7d0823 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -223,11 +223,7 @@ CopyToTextStart(CopyToState cstate, TupleDesc tupDesc) colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, colname, false, - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, colname); + CopyAttributeOutText(cstate, colname); } CopyToTextSendEndOfRow(cstate); @@ -260,12 +256,7 @@ CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot) char *string; string = OutputFunctionCall(&out_functions[attnum - 1], value); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, string, - cstate->opts.force_quote_flags[attnum - 1], - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, string); + CopyAttributeOutText(cstate, string); } } @@ -277,6 +268,99 @@ CopyToTextEnd(CopyToState cstate) { } +static void +CopyToCSVStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + /* + * For non-binary copy, we need to convert null_print to file encoding, + * because it will be sent directly with CopySendString. + */ + if (cstate->need_transcoding) + cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, + cstate->opts.null_print_len, + cstate->file_encoding); + + /* if a header has been requested send the line */ + if (cstate->opts.header_line) + { + bool hdr_delim = false; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + char *colname; + + if (hdr_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + hdr_delim = true; + + colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); + + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + } + + CopyToTextSendEndOfRow(cstate); + } +} + +static void +CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + bool need_delim = false; + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (need_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + need_delim = true; + + if (isnull) + { + CopySendString(cstate, cstate->opts.null_print_client); + } + else + { + char *string; + + string = OutputFunctionCall(&out_functions[attnum - 1], value); + CopyAttributeOutCSV(cstate, string, + cstate->opts.force_quote_flags[attnum - 1], + list_length(cstate->attnumlist) == 1); + } + } + + CopyToTextSendEndOfRow(cstate); +} + +static void +CopyToCSVEnd(CopyToState cstate) +{ +} + /* * CopyToRoutine implementation for "binary". */ @@ -388,9 +472,9 @@ CopyToRoutine CopyToRoutineText = { CopyToRoutine CopyToRoutineCSV = { .CopyToProcessOption = CopyToTextProcessOption, .CopyToGetFormat = CopyToTextGetFormat, - .CopyToStart = CopyToTextStart, - .CopyToOneRow = CopyToTextOneRow, - .CopyToEnd = CopyToTextEnd, + .CopyToStart = CopyToCSVStart, + .CopyToOneRow = CopyToCSVOneRow, + .CopyToEnd = CopyToCSVEnd, }; CopyToRoutine CopyToRoutineBinary = { ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 04:53 Michael Paquier <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-25 04:53 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote: > Hmm I can see a similar trend that Suto-san had; the binary format got > slightly faster whereas both text and csv format has small regression > (4%~5%). I think that the improvement for binary came from the fact > that we removed "if (cstate->opts.binary)" branches from the original > CopyOneRowTo(). I've experimented with a similar optimization for csv > and text format; have different callbacks for text and csv format and > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > for that. Here are results: > > HEAD w/ 0001 patch + remove branches: > binary 2824.502 ms > text 2715.264 ms > csv 2803.381 ms > > The numbers look better now. I'm not sure these are within a noise > range but it might be worth considering having different callbacks for > text and csv formats. Interesting. Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and 0.9% speedup for binary, which may be around the noise range assuming a ~1% range. While this does not imply a regression, that seems worth the duplication IMO. The patch had better document the reason why the split is done, as well. CopyFromTextOneRow() has also specific branches for binary and non-binary removed in 0005, so assuming that I/O is not a bottleneck, the operation would be faster because we would not evaluate this "if" condition for each row. Wouldn't we also see improvements for COPY FROM with short row values, say when mounting PGDATA into a tmpfs/ramfs? -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 05:28 Masahiko Sawada <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Masahiko Sawada @ 2024-01-25 05:28 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 1:53 PM Michael Paquier <[email protected]> wrote: > > On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote: > > Hmm I can see a similar trend that Suto-san had; the binary format got > > slightly faster whereas both text and csv format has small regression > > (4%~5%). I think that the improvement for binary came from the fact > > that we removed "if (cstate->opts.binary)" branches from the original > > CopyOneRowTo(). I've experimented with a similar optimization for csv > > and text format; have different callbacks for text and csv format and > > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > > for that. Here are results: > > > > HEAD w/ 0001 patch + remove branches: > > binary 2824.502 ms > > text 2715.264 ms > > csv 2803.381 ms > > > > The numbers look better now. I'm not sure these are within a noise > > range but it might be worth considering having different callbacks for > > text and csv formats. > > Interesting. > > Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and > 0.9% speedup for binary, which may be around the noise range assuming > a ~1% range. While this does not imply a regression, that seems worth > the duplication IMO. Agreed. In addition to that, now that each format routine has its own callbacks, there would be chances that we can do other optimizations dedicated to the format type in the future if available. > The patch had better document the reason why the > split is done, as well. +1 > > CopyFromTextOneRow() has also specific branches for binary and > non-binary removed in 0005, so assuming that I/O is not a bottleneck, > the operation would be faster because we would not evaluate this "if" > condition for each row. Wouldn't we also see improvements for COPY > FROM with short row values, say when mounting PGDATA into a > tmpfs/ramfs? Probably. Seems worth evaluating. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:05 Sutou Kouhei <[email protected]> parent: jian he <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:05 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, Thanks for trying these patches! In <CACJufxF9NS3xQ2d79jN0V1CGvF7cR16uJo-C3nrY7vZrwvxF7w@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 10:53:58 +0800, jian he <[email protected]> wrote: > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Wow! I didn't know the "\watch count="! I'll use it. > Time: 668.996 ms > Time: 596.254 ms > Time: 592.723 ms > Time: 591.663 ms > Time: 590.803 ms It seems that 5 times isn't enough for this case as Michael said. But thanks for trying! Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:45 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900, Michael Paquier <[email protected]> wrote: > +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); > +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); > +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); > +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); > +typedef void (*CopyToEnd_function) (CopyToState cstate); > > We don't really need a set of typedefs here, let's put the definitions > in the CopyToRoutine struct instead. OK. I'll do it. > +extern CopyToRoutine CopyToRoutineText; > +extern CopyToRoutine CopyToRoutineCSV; > +extern CopyToRoutine CopyToRoutineBinary; > > All that should IMO remain in copyto.c and copyfrom.c in the initial > patch doing the refactoring. Why not using a fetch function instead > that uses a string in input? Then you can call that once after > parsing the List of options in ProcessCopyOptions(). OK. How about the following for the fetch function signature? extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); We may introduce an enum and use it: typedef enum CopyBuiltinFormat { COPY_BUILTIN_FORMAT_TEXT = 0, COPY_BUILTIN_FORMAT_CSV, COPY_BUILTIN_FORMAT_BINARY, } CopyBuiltinFormat; extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format); > +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may > + * move the code to here later. */ > Some areas, like this comment, are written in an incorrect format. Oh, sorry. I assumed that the comment style was adjusted by pgindent. I'll use the following style: /* * ... */ > + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); > + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); > > Actually, this split is interesting. It is possible for a custom > format to plug in a custom set of out functions. Did you make use of > something custom for your own stuff? I didn't. My PoC custom COPY format handler for Apache Arrow just handles integer and text for now. It doesn't use cstate->out_functions because cstate->out_functions may not return a valid binary format value for Apache Arrow. So it formats each value by itself. I'll chose one of them for a custom type (that isn't supported by Apache Arrow, e.g. PostGIS types): 1. Report an unsupported error 2. Call output function for Apache Arrow provided by the custom type > Actually, could it make sense to > split the assignment of cstate->out_functions into its own callback? Yes. Because we need to use getTypeBinaryOutputInfo() for "binary" and use getTypeOutputInfo() for "text" and "csv". > Sure, that's part of the start phase, but at least it would make clear > that a custom method *has* to assign these OIDs to work. The patch > implies that as a rule, without a comment that CopyToStart *must* set > up these OIDs. CopyToStart doesn't need to set up them if the handler doesn't use cstate->out_functions. > I think that 0001 and 0005 should be handled first, as pieces > independent of the rest. Then we could move on with 0002~0004 and > 0006~0008. OK. I'll focus on 0001 and 0005 for now. I'll restart 0002-0004/0006-0008 after 0001 and 0005 are accepted. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:52 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:52 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900, Masahiko Sawada <[email protected]> wrote: > I've experimented with a similar optimization for csv > and text format; have different callbacks for text and csv format and > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > for that. Here are results: > > HEAD w/ 0001 patch + remove branches: > binary 2824.502 ms > text 2715.264 ms > csv 2803.381 ms > > The numbers look better now. I'm not sure these are within a noise > range but it might be worth considering having different callbacks for > text and csv formats. Wow! Interesting. I tried the approach before but I didn't see any difference by the approach. But it may depend on my environment. I'll import the approach to the next patch set so that others can try the approach easily. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 23:35 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-25 23:35 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 05:45:43PM +0900, Sutou Kouhei wrote: > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900, > Michael Paquier <[email protected]> wrote: >> +extern CopyToRoutine CopyToRoutineText; >> +extern CopyToRoutine CopyToRoutineCSV; >> +extern CopyToRoutine CopyToRoutineBinary; >> >> All that should IMO remain in copyto.c and copyfrom.c in the initial >> patch doing the refactoring. Why not using a fetch function instead >> that uses a string in input? Then you can call that once after >> parsing the List of options in ProcessCopyOptions(). > > OK. How about the following for the fetch function > signature? > > extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad history with naming things around here. > We may introduce an enum and use it: > > typedef enum CopyBuiltinFormat > { > COPY_BUILTIN_FORMAT_TEXT = 0, > COPY_BUILTIN_FORMAT_CSV, > COPY_BUILTIN_FORMAT_BINARY, > } CopyBuiltinFormat; > > extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format); I am not sure that this is necessary as the option value is a string. > Oh, sorry. I assumed that the comment style was adjusted by > pgindent. No worries, that's just something we get used to. I tend to fix a lot of these things by myself when editing patches. >> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); >> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); >> >> Actually, this split is interesting. It is possible for a custom >> format to plug in a custom set of out functions. Did you make use of >> something custom for your own stuff? > > I didn't. My PoC custom COPY format handler for Apache Arrow > just handles integer and text for now. It doesn't use > cstate->out_functions because cstate->out_functions may not > return a valid binary format value for Apache Arrow. So it > formats each value by itself. I mean, if you use a custom output function, you could tweak things even more with byteas or such.. If a callback is expected to do something, like setting the output function OIDs in the start callback, we'd better document it rather than letting that be implied. >> Actually, could it make sense to >> split the assignment of cstate->out_functions into its own callback? > > Yes. Because we need to use getTypeBinaryOutputInfo() for > "binary" and use getTypeOutputInfo() for "text" and "csv". Okay. After sleeping on it, a split makes sense here, because it also reduces the presence of TupleDesc in the start callback. >> Sure, that's part of the start phase, but at least it would make clear >> that a custom method *has* to assign these OIDs to work. The patch >> implies that as a rule, without a comment that CopyToStart *must* set >> up these OIDs. > > CopyToStart doesn't need to set up them if the handler > doesn't use cstate->out_functions. Noted. >> I think that 0001 and 0005 should be handled first, as pieces >> independent of the rest. Then we could move on with 0002~0004 and >> 0006~0008. > > OK. I'll focus on 0001 and 0005 for now. I'll restart > 0002-0004/0006-0008 after 0001 and 0005 are accepted. Once you get these, I'd be interested in re-doing an evaluation of COPY TO and more tests with COPY FROM while running Postgres on scissors. One thing I was thinking to use here is my blackhole_am for COPY FROM: https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am As per its name, it does nothing on INSERT, so you could create a table using it as access method, and stress the COPY FROM execution paths without having to mount Postgres on a tmpfs because the data is sent to the void. Perhaps it does not matter, but that moves the tests to the bottlenecks we want to stress (aka the per-row callback for large data sets). I've switched the patch as waiting on author for now. Thanks for your perseverance here. I understand that's not easy to follow up with patches and reviews (^_^;) -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:18 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Junwang Zhao @ 2024-01-26 08:18 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 4:52 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900, > Masahiko Sawada <[email protected]> wrote: > > > I've experimented with a similar optimization for csv > > and text format; have different callbacks for text and csv format and > > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > > for that. Here are results: > > > > HEAD w/ 0001 patch + remove branches: > > binary 2824.502 ms > > text 2715.264 ms > > csv 2803.381 ms > > > > The numbers look better now. I'm not sure these are within a noise > > range but it might be worth considering having different callbacks for > > text and csv formats. > > Wow! Interesting. I tried the approach before but I didn't > see any difference by the approach. But it may depend on my > environment. > > I'll import the approach to the next patch set so that > others can try the approach easily. > > > Thanks, > -- > kou Hi Kou-san, In the current implementation, there is no way that one can check incompatibility options in ProcessCopyOptions, we can postpone the check in CopyFromStart or CopyToStart, but I think it is a little bit late. Do you think adding an extra check for incompatible options hook is acceptable (PFA)? -- Regards Junwang Zhao Attachments: [application/octet-stream] 0001-add-check-incomptiblity-options-hooks.patch (3.1K, ../../CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com/2-0001-add-check-incomptiblity-options-hooks.patch) download | inline diff: From f01ad61fe6c251a00870ace5d37669350f8e2043 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Fri, 26 Jan 2024 15:55:07 +0800 Subject: [PATCH] add check incomptiblity options hooks Signed-off-by: Zhao Junwang <[email protected]> --- src/backend/commands/copy.c | 5 +++++ src/include/commands/copyapi.h | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 479f36868c..985d50870f 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -924,6 +924,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("NULL specification and DEFAULT specification cannot be the same"))); } + + if (is_from) + opts_out->from_routine->CopyFromCheckIncompatibleOptions(cstate, opts_out); + else + opts_out->to_routine->CopyToCheckIncompatibleOptions(cstate, opts_out); } /* diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 22accc83ab..34aa86761a 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -21,8 +21,10 @@ #include "nodes/parsenodes.h" typedef struct CopyFromStateData *CopyFromState; +typedef struct CopyFormatOptions; typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); +typedef void (*CopyFromCheckIncompatibleOptions_function) (CopyFromState cstate, CopyFormatOptions *options); typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate); typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc); typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); @@ -39,6 +41,12 @@ typedef struct CopyFromRoutine */ CopyFromProcessOption_function CopyFromProcessOption; + /* + * Called for processing incompatible options. This will report error + * messages when find incompatible options. + */ + CopyFromCheckIncompatibleOptions_function CopyFromCheckIncompatibleOptions; + /* * Called when COPY FROM is started. This will return a format as int16 * value. It's used for the CopyInResponse message. @@ -67,6 +75,7 @@ extern CopyFromRoutine CopyFromRoutineBinary; typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef void (*CopyToCheckIncompatibleOptions_function) (CopyToState cstate, CopyFormatOptions *options); typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); @@ -83,6 +92,12 @@ typedef struct CopyToRoutine */ CopyToProcessOption_function CopyToProcessOption; + /* + * Called for processing incompatible options. This will report error + * messages when find incompatible options. + */ + CopyToCheckIncompatibleOptions_function CopyToCheckIncompatibleOptions; + /* * Called when COPY TO is started. This will return a format as int16 * value. It's used for the CopyOutResponse message. -- 2.41.0 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:32 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:32 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800, Junwang Zhao <[email protected]> wrote: > In the current implementation, there is no way that one can check > incompatibility > options in ProcessCopyOptions, we can postpone the check in CopyFromStart > or CopyToStart, but I think it is a little bit late. Do you think > adding an extra > check for incompatible options hook is acceptable (PFA)? Thanks for the suggestion! But I think that a custom handler can do it in CopyToProcessOption()/CopyFromProcessOption(). What do you think about this? Or could you share a sample COPY TO/FROM WITH() SQL you think? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:41 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Junwang Zhao @ 2024-01-26 08:41 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 4:32 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800, > Junwang Zhao <[email protected]> wrote: > > > In the current implementation, there is no way that one can check > > incompatibility > > options in ProcessCopyOptions, we can postpone the check in CopyFromStart > > or CopyToStart, but I think it is a little bit late. Do you think > > adding an extra > > check for incompatible options hook is acceptable (PFA)? > > Thanks for the suggestion! But I think that a custom handler > can do it in > CopyToProcessOption()/CopyFromProcessOption(). What do you > think about this? Or could you share a sample COPY TO/FROM > WITH() SQL you think? CopyToProcessOption()/CopyFromProcessOption() can only handle single option, and store the options in the opaque field, but it can not check the relation of two options, for example, considering json format, the `header` option can not be handled by these two functions. I want to find a way when the user specifies the header option, customer handler can error out. > > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:49 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:49 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 08:35:19 +0900, Michael Paquier <[email protected]> wrote: >> OK. How about the following for the fetch function >> signature? >> >> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); > > Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad > history with naming things around here. Thanks for the suggestion. I rethink about this and use the following: +extern void ProcessCopyOptionFormatTo(ParseState *pstate, CopyFormatOptions *opts_out, DefElem *defel); It's not a fetch function. It sets CopyToRoutine opts_out instead. But it hides CopyToRoutine* to copyto.c. Is it acceptable? >> OK. I'll focus on 0001 and 0005 for now. I'll restart >> 0002-0004/0006-0008 after 0001 and 0005 are accepted. > > Once you get these, I'd be interested in re-doing an evaluation of > COPY TO and more tests with COPY FROM while running Postgres on > scissors. One thing I was thinking to use here is my blackhole_am for > COPY FROM: > https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am Thanks! Could you evaluate the attached patch set with COPY FROM? I attach v7 patch set. It includes only the 0001 and 0005 parts in v6 patch set because we focus on them for now. 0001: This is based on 0001 in v6. Changes since v6: * Fix comment style * Hide CopyToRoutine{Text,CSV,Binary} * Add more comments * Eliminate "if (cstate->opts.csv_mode)" branches from "text" and "csv" callbacks * Remove CopyTo*_function typedefs * Update benchmark results in commit message but the results are measured on my environment that isn't suitable for accurate benchmark 0002: This is based on 0005 in v6. Changes since v6: * Fix comment style * Hide CopyFromRoutine{Text,CSV,Binary} * Add more comments * Eliminate a "if (cstate->opts.csv_mode)" branch from "text" and "csv" callbacks * NOTE: We can eliminate more "if (cstate->opts.csv_mode)" branches such as one in NextCopyFromRawFields(). Should we do it in this feature improvement (make COPY format extendable)? Can we defer this as a separated improvement? * Remove CopyFrom*_function typedefs Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:55 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:55 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, Junwang Zhao <[email protected]> wrote: > CopyToProcessOption()/CopyFromProcessOption() can only handle > single option, and store the options in the opaque field, but it can not > check the relation of two options, for example, considering json format, > the `header` option can not be handled by these two functions. > > I want to find a way when the user specifies the header option, customer > handler can error out. Ah, you want to use a built-in option (such as "header") value from a custom handler, right? Hmm, it may be better that we call CopyToProcessOption()/CopyFromProcessOption() for all options including built-in options. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 09:02 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Junwang Zhao @ 2024-01-26 09:02 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > Junwang Zhao <[email protected]> wrote: > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > single option, and store the options in the opaque field, but it can not > > check the relation of two options, for example, considering json format, > > the `header` option can not be handled by these two functions. > > > > I want to find a way when the user specifies the header option, customer > > handler can error out. > > Ah, you want to use a built-in option (such as "header") > value from a custom handler, right? Hmm, it may be better > that we call CopyToProcessOption()/CopyFromProcessOption() > for all options including built-in options. > Hmm, still I don't think it can handle all cases, since we don't know the sequence of the options, we need all the options been parsed before we check the compatibility of the options, or customer handlers will need complicated logic to resolve that, which might lead to ugly code :( > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-27 06:15 Junwang Zhao <[email protected]> parent: Junwang Zhao <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Junwang Zhao @ 2024-01-27 06:15 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi Kou-san, On Fri, Jan 26, 2024 at 5:02 PM Junwang Zhao <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > Junwang Zhao <[email protected]> wrote: > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > single option, and store the options in the opaque field, but it can not > > > check the relation of two options, for example, considering json format, > > > the `header` option can not be handled by these two functions. > > > > > > I want to find a way when the user specifies the header option, customer > > > handler can error out. > > > > Ah, you want to use a built-in option (such as "header") > > value from a custom handler, right? Hmm, it may be better > > that we call CopyToProcessOption()/CopyFromProcessOption() > > for all options including built-in options. > > > Hmm, still I don't think it can handle all cases, since we don't know > the sequence of the options, we need all the options been parsed > before we check the compatibility of the options, or customer > handlers will need complicated logic to resolve that, which might > lead to ugly code :( > I have been working on a *COPY TO JSON* extension since yesterday, which is based on your V6 patch set, I'd like to give you more input so you can make better decisions about the implementation(with only pg-copy-arrow you might not get everything considered). V8 is based on V6, so anybody involved in the performance issue should still review the V7 patch set. 0001-0008 is your original V6 implementations 0009 is some changes made by me, I changed CopyToGetFormat to CopyToSendCopyBegin because pg_copy_json need to send different bytes in SendCopyBegin, get the format code along is not enough, I once had a thought that may be we should merge SendCopyBegin/SendCopyEnd into CopyToStart/CopyToEnd but I don't do that in this patch. I have also exported more APIs for extension usage. 00010 is the pg_copy_json extension, I think this should be a good case which can utilize the *extendable copy format* feature, maybe we should delete copy_test_format if we have this extension as an example? > > > > Thanks, > > -- > > kou > > > > -- > Regards > Junwang Zhao -- Regards Junwang Zhao Attachments: [application/octet-stream] v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch (3.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/2-v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch) download | inline diff: From 4b177469f3fb8f14f0cd6bff3c7878dcafd9b760 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 15:12:43 +0900 Subject: [PATCH v8 04/10] Add support for implementing custom COPY TO format as extension * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). --- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cfc74ee7b1..b5d8678394 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) default: break; } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } CopyToRoutine CopyToRoutineText = { @@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index a869d78d72..ffad433a21 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -174,6 +174,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0001-Extract-COPY-TO-format-implementations.patch (23.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/3-v8-0001-Extract-COPY-TO-format-implementations.patch) download | inline diff: From 6e68ba6380dc825a242e7f0d0a53442bba3a4a61 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Mon, 4 Dec 2023 12:32:54 +0900 Subject: [PATCH v8 01/10] Extract COPY TO format implementations This is a part of making COPY format extendable. See also these past discussions: * New Copy Formats - avro/orc/parquet: https://www.postgresql.org/message-id/flat/20180210151304.fonjztsynewldfba%40gmail.com * Make COPY extendable in order to support Parquet and other formats: https://www.postgresql.org/message-id/flat/CAJ7c6TM6Bz1c3F04Cy6%2BSzuWfKmr0kU8c_3Stnvh_8BR0D6k8Q%40mail.gmail.com This doesn't change the current behavior. This just introduces CopyToRoutine, which just has function pointers of format implementation like TupleTableSlotOps, and use it for existing "text", "csv" and "binary" format implementations. Note that CopyToRoutine can't be used from extensions yet because CopySend*() aren't exported yet. Extensions can't send formatted data to a destination without CopySend*(). They will be exported by subsequent patches. Here is a benchmark result with/without this change because there was a discussion that we should care about performance regression: https://www.postgresql.org/message-id/3741749.1655952719%40sss.pgh.pa.us > I think that step 1 ought to be to convert the existing formats into > plug-ins, and demonstrate that there's no significant loss of > performance. You can see that there is no significant loss of performance: Data: Random 32 bit integers: CREATE TABLE data (int32 integer); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${n_records}); The number of records: 100K, 1M and 10M 100K without this change: format,elapsed time (ms) text,11.002 csv,11.696 binary,11.352 100K with this change: format,elapsed time (ms) text,100000,11.562 csv,100000,11.889 binary,100000,10.825 1M without this change: format,elapsed time (ms) text,108.359 csv,114.233 binary,111.251 1M with this change: format,elapsed time (ms) text,111.269 csv,116.277 binary,104.765 10M without this change: format,elapsed time (ms) text,1090.763 csv,1136.103 binary,1137.141 10M with this change: format,elapsed time (ms) text,1082.654 csv,1196.991 binary,1069.697 --- contrib/file_fdw/file_fdw.c | 2 +- src/backend/commands/copy.c | 43 +++- src/backend/commands/copyfrom.c | 2 +- src/backend/commands/copyto.c | 428 ++++++++++++++++++++------------ src/include/commands/copy.h | 7 +- src/include/commands/copyapi.h | 59 +++++ 6 files changed, 376 insertions(+), 165 deletions(-) create mode 100644 src/include/commands/copyapi.h diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 249d82d3a0..9e4e819858 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -329,7 +329,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Now apply the core COPY code's validation logic for more checks. */ - ProcessCopyOptions(NULL, NULL, true, other_options); + ProcessCopyOptions(NULL, NULL, true, NULL, other_options); /* * Either filename or program option is required for file_fdw foreign diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index cc0786c6f4..5f3697a5f9 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -442,6 +442,9 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from) * a list of options. In that usage, 'opts_out' can be passed as NULL and * the collected data is just leaked until CurrentMemoryContext is reset. * + * 'cstate' is CopyToState* for !is_from, CopyFromState* for is_from. 'cstate' + * may be NULL. For example, file_fdw uses NULL. + * * Note that additional checking, such as whether column names listed in FORCE * QUOTE actually exist, has to be applied later. This just checks for * self-consistency of the options list. @@ -450,6 +453,7 @@ void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, + void *cstate, List *options) { bool format_specified = false; @@ -464,7 +468,13 @@ ProcessCopyOptions(ParseState *pstate, opts_out->file_encoding = -1; - /* Extract options from the statement node tree */ + /* Text is the default format. */ + opts_out->to_routine = &CopyToRoutineText; + + /* + * Extract only the "format" option to detect target routine as the first + * step + */ foreach(option, options) { DefElem *defel = lfirst_node(DefElem, option); @@ -479,15 +489,29 @@ ProcessCopyOptions(ParseState *pstate, if (strcmp(fmt, "text") == 0) /* default format */ ; else if (strcmp(fmt, "csv") == 0) + { opts_out->csv_mode = true; + opts_out->to_routine = &CopyToRoutineCSV; + } else if (strcmp(fmt, "binary") == 0) + { opts_out->binary = true; + opts_out->to_routine = &CopyToRoutineBinary; + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY format \"%s\" not recognized", fmt), parser_errposition(pstate, defel->location))); } + } + /* Extract options except "format" from the statement node tree */ + foreach(option, options) + { + DefElem *defel = lfirst_node(DefElem, option); + + if (strcmp(defel->defname, "format") == 0) + continue; else if (strcmp(defel->defname, "freeze") == 0) { if (freeze_specified) @@ -616,11 +640,18 @@ ProcessCopyOptions(ParseState *pstate, opts_out->on_error = defGetCopyOnErrorChoice(defel, pstate, is_from); } else - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", - defel->defname), - parser_errposition(pstate, defel->location))); + { + bool processed = false; + + if (!is_from) + processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); + if (!processed) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("option \"%s\" not recognized", + defel->defname), + parser_errposition(pstate, defel->location))); + } } /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 1fe70b9133..fb3d4d9296 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1416,7 +1416,7 @@ BeginCopyFrom(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options); + ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , cstate, options); /* Process the target relation */ cstate->rel = rel; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index d3dc3fc854..6547b7c654 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -131,6 +131,275 @@ static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); +/* + * CopyToRoutine implementations. + */ + +/* + * CopyToRoutine implementation for "text" and "csv". CopyToText*() + * refer cstate->opts.csv_mode and change their behavior. We can split this + * implementation and stop referring cstate->opts.csv_mode later. + */ + +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ +static bool +CopyToTextProcessOption(CopyToState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyToTextGetFormat(CopyToState cstate) +{ + return 0; +} + +static void +CopyToTextSendEndOfRow(CopyToState cstate) +{ + switch (cstate->copy_dest) + { + case COPY_FILE: + /* Default line termination depends on platform */ +#ifndef WIN32 + CopySendChar(cstate, '\n'); +#else + CopySendString(cstate, "\r\n"); +#endif + break; + case COPY_FRONTEND: + /* The FE/BE protocol uses \n as newline for all platforms */ + CopySendChar(cstate, '\n'); + break; + default: + break; + } + CopySendEndOfRow(cstate); +} + +static void +CopyToTextStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + /* + * For non-binary copy, we need to convert null_print to file encoding, + * because it will be sent directly with CopySendString. + */ + if (cstate->need_transcoding) + cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, + cstate->opts.null_print_len, + cstate->file_encoding); + + /* if a header has been requested send the line */ + if (cstate->opts.header_line) + { + bool hdr_delim = false; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + char *colname; + + if (hdr_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + hdr_delim = true; + + colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); + + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, colname); + } + + CopyToTextSendEndOfRow(cstate); + } +} + +static void +CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + bool need_delim = false; + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (need_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + need_delim = true; + + if (isnull) + { + CopySendString(cstate, cstate->opts.null_print_client); + } + else + { + char *string; + + string = OutputFunctionCall(&out_functions[attnum - 1], value); + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, string, + cstate->opts.force_quote_flags[attnum - 1], + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, string); + } + } + + CopyToTextSendEndOfRow(cstate); +} + +static void +CopyToTextEnd(CopyToState cstate) +{ +} + +/* + * CopyToRoutine implementation for "binary". + */ + +/* All "binary" options are parsed in ProcessCopyOptions(). We may move the + * code to here later. */ +static bool +CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyToBinaryGetFormat(CopyToState cstate) +{ + return 1; +} + +static void +CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + { + /* Generate header for a binary copy */ + int32 tmp; + + /* Signature */ + CopySendData(cstate, BinarySignature, 11); + /* Flags field */ + tmp = 0; + CopySendInt32(cstate, tmp); + /* No header extension */ + tmp = 0; + CopySendInt32(cstate, tmp); + } +} + +static void +CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + /* Binary per-tuple header */ + CopySendInt16(cstate, list_length(cstate->attnumlist)); + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (isnull) + { + CopySendInt32(cstate, -1); + } + else + { + bytea *outputbytes; + + outputbytes = SendFunctionCall(&out_functions[attnum - 1], value); + CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ); + CopySendData(cstate, VARDATA(outputbytes), + VARSIZE(outputbytes) - VARHDRSZ); + } + } + + CopySendEndOfRow(cstate); +} + +static void +CopyToBinaryEnd(CopyToState cstate) +{ + /* Generate trailer for a binary copy */ + CopySendInt16(cstate, -1); + /* Need to flush out the trailer */ + CopySendEndOfRow(cstate); +} + +CopyToRoutine CopyToRoutineText = { + .CopyToProcessOption = CopyToTextProcessOption, + .CopyToGetFormat = CopyToTextGetFormat, + .CopyToStart = CopyToTextStart, + .CopyToOneRow = CopyToTextOneRow, + .CopyToEnd = CopyToTextEnd, +}; + +/* + * We can use the same CopyToRoutine for both of "text" and "csv" because + * CopyToText*() refer cstate->opts.csv_mode and change their behavior. We can + * split the implementations and stop referring cstate->opts.csv_mode later. + */ +CopyToRoutine CopyToRoutineCSV = { + .CopyToProcessOption = CopyToTextProcessOption, + .CopyToGetFormat = CopyToTextGetFormat, + .CopyToStart = CopyToTextStart, + .CopyToOneRow = CopyToTextOneRow, + .CopyToEnd = CopyToTextEnd, +}; + +CopyToRoutine CopyToRoutineBinary = { + .CopyToProcessOption = CopyToBinaryProcessOption, + .CopyToGetFormat = CopyToBinaryGetFormat, + .CopyToStart = CopyToBinaryStart, + .CopyToOneRow = CopyToBinaryOneRow, + .CopyToEnd = CopyToBinaryEnd, +}; /* * Send copy start/stop messages for frontend copies. These have changed @@ -141,7 +410,7 @@ SendCopyBegin(CopyToState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate); int i; pq_beginmessage(&buf, PqMsg_CopyOutResponse); @@ -198,16 +467,6 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { case COPY_FILE: - if (!cstate->opts.binary) - { - /* Default line termination depends on platform */ -#ifndef WIN32 - CopySendChar(cstate, '\n'); -#else - CopySendString(cstate, "\r\n"); -#endif - } - if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || ferror(cstate->copy_file)) @@ -242,10 +501,6 @@ CopySendEndOfRow(CopyToState cstate) } break; case COPY_FRONTEND: - /* The FE/BE protocol uses \n as newline for all platforms */ - if (!cstate->opts.binary) - CopySendChar(cstate, '\n'); - /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; @@ -431,7 +686,7 @@ BeginCopyTo(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options); + ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , cstate, options); /* Process the source/target relation or query */ if (rel) @@ -748,8 +1003,6 @@ DoCopyTo(CopyToState cstate) bool pipe = (cstate->filename == NULL && cstate->data_dest_cb == NULL); bool fe_copy = (pipe && whereToSendOutput == DestRemote); TupleDesc tupDesc; - int num_phys_attrs; - ListCell *cur; uint64 processed; if (fe_copy) @@ -759,32 +1012,11 @@ DoCopyTo(CopyToState cstate) tupDesc = RelationGetDescr(cstate->rel); else tupDesc = cstate->queryDesc->tupDesc; - num_phys_attrs = tupDesc->natts; cstate->opts.null_print_client = cstate->opts.null_print; /* default */ /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ cstate->fe_msgbuf = makeStringInfo(); - /* Get info about the columns we need to process. */ - cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - Oid out_func_oid; - bool isvarlena; - Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - - if (cstate->opts.binary) - getTypeBinaryOutputInfo(attr->atttypid, - &out_func_oid, - &isvarlena); - else - getTypeOutputInfo(attr->atttypid, - &out_func_oid, - &isvarlena); - fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); - } - /* * Create a temporary memory context that we can reset once per row to * recover palloc'd memory. This avoids any problems with leaks inside @@ -795,57 +1027,7 @@ DoCopyTo(CopyToState cstate) "COPY TO", ALLOCSET_DEFAULT_SIZES); - if (cstate->opts.binary) - { - /* Generate header for a binary copy */ - int32 tmp; - - /* Signature */ - CopySendData(cstate, BinarySignature, 11); - /* Flags field */ - tmp = 0; - CopySendInt32(cstate, tmp); - /* No header extension */ - tmp = 0; - CopySendInt32(cstate, tmp); - } - else - { - /* - * For non-binary copy, we need to convert null_print to file - * encoding, because it will be sent directly with CopySendString. - */ - if (cstate->need_transcoding) - cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, - cstate->opts.null_print_len, - cstate->file_encoding); - - /* if a header has been requested send the line */ - if (cstate->opts.header_line) - { - bool hdr_delim = false; - - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - char *colname; - - if (hdr_delim) - CopySendChar(cstate, cstate->opts.delim[0]); - hdr_delim = true; - - colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, colname, false, - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, colname); - } - - CopySendEndOfRow(cstate); - } - } + cstate->opts.to_routine->CopyToStart(cstate, tupDesc); if (cstate->rel) { @@ -884,13 +1066,7 @@ DoCopyTo(CopyToState cstate) processed = ((DR_copy *) cstate->queryDesc->dest)->processed; } - if (cstate->opts.binary) - { - /* Generate trailer for a binary copy */ - CopySendInt16(cstate, -1); - /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); - } + cstate->opts.to_routine->CopyToEnd(cstate); MemoryContextDelete(cstate->rowcontext); @@ -906,71 +1082,15 @@ DoCopyTo(CopyToState cstate) static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) { - bool need_delim = false; - FmgrInfo *out_functions = cstate->out_functions; MemoryContext oldcontext; - ListCell *cur; - char *string; MemoryContextReset(cstate->rowcontext); oldcontext = MemoryContextSwitchTo(cstate->rowcontext); - if (cstate->opts.binary) - { - /* Binary per-tuple header */ - CopySendInt16(cstate, list_length(cstate->attnumlist)); - } - /* Make sure the tuple is fully deconstructed */ slot_getallattrs(slot); - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - Datum value = slot->tts_values[attnum - 1]; - bool isnull = slot->tts_isnull[attnum - 1]; - - if (!cstate->opts.binary) - { - if (need_delim) - CopySendChar(cstate, cstate->opts.delim[0]); - need_delim = true; - } - - if (isnull) - { - if (!cstate->opts.binary) - CopySendString(cstate, cstate->opts.null_print_client); - else - CopySendInt32(cstate, -1); - } - else - { - if (!cstate->opts.binary) - { - string = OutputFunctionCall(&out_functions[attnum - 1], - value); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, string, - cstate->opts.force_quote_flags[attnum - 1], - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, string); - } - else - { - bytea *outputbytes; - - outputbytes = SendFunctionCall(&out_functions[attnum - 1], - value); - CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ); - CopySendData(cstate, VARDATA(outputbytes), - VARSIZE(outputbytes) - VARHDRSZ); - } - } - } - - CopySendEndOfRow(cstate); + cstate->opts.to_routine->CopyToOneRow(cstate, slot); MemoryContextSwitchTo(oldcontext); } diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index b3da3cb0be..34bea880ca 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -14,6 +14,7 @@ #ifndef COPY_H #define COPY_H +#include "commands/copyapi.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -74,11 +75,11 @@ typedef struct CopyFormatOptions bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ List *convert_select; /* list of column names (can be NIL) */ + CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; -/* These are private in commands/copy[from|to].c */ +/* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; -typedef struct CopyToStateData *CopyToState; typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); typedef void (*copy_data_dest_cb) (void *data, int len); @@ -87,7 +88,7 @@ extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); -extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options); +extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options); extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *whereClause, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h new file mode 100644 index 0000000000..eb68f2fb7b --- /dev/null +++ b/src/include/commands/copyapi.h @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------- + * + * copyapi.h + * API for COPY TO/FROM handlers + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/commands/copyapi.h + * + *------------------------------------------------------------------------- + */ +#ifndef COPYAPI_H +#define COPYAPI_H + +#include "executor/tuptable.h" +#include "nodes/parsenodes.h" + +/* This is private in commands/copyto.c */ +typedef struct CopyToStateData *CopyToState; + +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); +typedef void (*CopyToEnd_function) (CopyToState cstate); + +/* Routines for a COPY TO format implementation. */ +typedef struct CopyToRoutine +{ + /* + * Called for processing one COPY TO option. This will return false when + * the given option is invalid. + */ + CopyToProcessOption_function CopyToProcessOption; + + /* + * Called when COPY TO is started. This will return a format as int16 + * value. It's used for the CopyOutResponse message. + */ + CopyToGetFormat_function CopyToGetFormat; + + /* Called when COPY TO is started. This will send a header. */ + CopyToStart_function CopyToStart; + + /* Copy one row for COPY TO. */ + CopyToOneRow_function CopyToOneRow; + + /* Called when COPY TO is ended. This will send a trailer. */ + CopyToEnd_function CopyToEnd; +} CopyToRoutine; + +/* Built-in CopyToRoutine for "text", "csv" and "binary". */ +extern CopyToRoutine CopyToRoutineText; +extern CopyToRoutine CopyToRoutineCSV; +extern CopyToRoutine CopyToRoutineBinary; + +#endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch (17.7K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/4-v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch) download | inline diff: From a597f8a2beec12971d77419f08b5722f531774f3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 13:58:38 +0900 Subject: [PATCH v8 02/10] Add support for adding custom COPY TO format This uses the handler approach like tablesample. The approach creates an internal function that returns an internal struct. In this case, a COPY TO handler returns a CopyToRoutine. We will add support for custom COPY FROM format later. We'll use the same handler for COPY TO and COPY FROM. PostgreSQL calls a COPY TO/FROM handler with "is_from" argument. It's true for COPY FROM and false for COPY TO: copy_handler(true) returns CopyToRoutine copy_handler(false) returns CopyFromRoutine (not exist yet) We discussed that we introduce a wrapper struct for it: typedef struct CopyRoutine { NodeTag type; /* either CopyToRoutine or CopyFromRoutine */ Node *routine; } copy_handler(true) returns CopyRoutine with CopyToRoutine copy_handler(false) returns CopyRoutine with CopyFromRoutine See also: https://www.postgresql.org/message-id/flat/CAD21AoCunywHird3GaPzWe6s9JG1wzxj3Cr6vGN36DDheGjOjA%40mail.gmail.com But I noticed that we don't need the wrapper struct. We can just CopyToRoutine or CopyFromRoutine. Because we can distinct the returned struct by checking its NodeTag. So I don't use the wrapper struct approach. --- src/backend/commands/copy.c | 84 ++++++++++++++----- src/backend/nodes/Makefile | 1 + src/backend/nodes/gen_node_support.pl | 2 + src/backend/utils/adt/pseudotypes.c | 1 + src/include/catalog/pg_proc.dat | 6 ++ src/include/catalog/pg_type.dat | 6 ++ src/include/commands/copyapi.h | 2 + src/include/nodes/meson.build | 1 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_copy_format/.gitignore | 4 + src/test/modules/test_copy_format/Makefile | 23 +++++ .../expected/test_copy_format.out | 17 ++++ src/test/modules/test_copy_format/meson.build | 33 ++++++++ .../test_copy_format/sql/test_copy_format.sql | 8 ++ .../test_copy_format--1.0.sql | 8 ++ .../test_copy_format/test_copy_format.c | 77 +++++++++++++++++ .../test_copy_format/test_copy_format.control | 4 + 18 files changed, 260 insertions(+), 19 deletions(-) mode change 100644 => 100755 src/backend/nodes/gen_node_support.pl create mode 100644 src/test/modules/test_copy_format/.gitignore create mode 100644 src/test/modules/test_copy_format/Makefile create mode 100644 src/test/modules/test_copy_format/expected/test_copy_format.out create mode 100644 src/test/modules/test_copy_format/meson.build create mode 100644 src/test/modules/test_copy_format/sql/test_copy_format.sql create mode 100644 src/test/modules/test_copy_format/test_copy_format--1.0.sql create mode 100644 src/test/modules/test_copy_format/test_copy_format.c create mode 100644 src/test/modules/test_copy_format/test_copy_format.control diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 5f3697a5f9..6f0db0ae7c 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -32,6 +32,7 @@ #include "parser/parse_coerce.h" #include "parser/parse_collate.h" #include "parser/parse_expr.h" +#include "parser/parse_func.h" #include "parser/parse_relation.h" #include "rewrite/rewriteHandler.h" #include "utils/acl.h" @@ -430,6 +431,69 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from) return COPY_ON_ERROR_STOP; /* keep compiler quiet */ } +/* + * Process the "format" option. + * + * This function checks whether the option value is a built-in format such as + * "text" and "csv" or not. If the option value isn't a built-in format, this + * function finds a COPY format handler that returns a CopyToRoutine. If no + * COPY format handler is found, this function reports an error. + */ +static void +ProcessCopyOptionCustomFormat(ParseState *pstate, + CopyFormatOptions *opts_out, + bool is_from, + DefElem *defel) +{ + char *format; + Oid funcargtypes[1]; + Oid handlerOid = InvalidOid; + Datum datum; + void *routine; + + format = defGetString(defel); + + /* built-in formats */ + if (strcmp(format, "text") == 0) + /* default format */ return; + else if (strcmp(format, "csv") == 0) + { + opts_out->csv_mode = true; + opts_out->to_routine = &CopyToRoutineCSV; + return; + } + else if (strcmp(format, "binary") == 0) + { + opts_out->binary = true; + opts_out->to_routine = &CopyToRoutineBinary; + return; + } + + /* custom format */ + if (!is_from) + { + funcargtypes[0] = INTERNALOID; + handlerOid = LookupFuncName(list_make1(makeString(format)), 1, + funcargtypes, true); + } + if (!OidIsValid(handlerOid)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", format), + parser_errposition(pstate, defel->location))); + + datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from)); + routine = DatumGetPointer(datum); + if (routine == NULL || !IsA(routine, CopyToRoutine)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct", + format, handlerOid), + parser_errposition(pstate, defel->location))); + + opts_out->to_routine = routine; +} + /* * Process the statement option list for COPY. * @@ -481,28 +545,10 @@ ProcessCopyOptions(ParseState *pstate, if (strcmp(defel->defname, "format") == 0) { - char *fmt = defGetString(defel); - if (format_specified) errorConflictingDefElem(defel, pstate); format_specified = true; - if (strcmp(fmt, "text") == 0) - /* default format */ ; - else if (strcmp(fmt, "csv") == 0) - { - opts_out->csv_mode = true; - opts_out->to_routine = &CopyToRoutineCSV; - } - else if (strcmp(fmt, "binary") == 0) - { - opts_out->binary = true; - opts_out->to_routine = &CopyToRoutineBinary; - } - else - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt), - parser_errposition(pstate, defel->location))); + ProcessCopyOptionCustomFormat(pstate, opts_out, is_from, defel); } } /* Extract options except "format" from the statement node tree */ diff --git a/src/backend/nodes/Makefile b/src/backend/nodes/Makefile index 66bbad8e6e..173ee11811 100644 --- a/src/backend/nodes/Makefile +++ b/src/backend/nodes/Makefile @@ -49,6 +49,7 @@ node_headers = \ access/sdir.h \ access/tableam.h \ access/tsmapi.h \ + commands/copyapi.h \ commands/event_trigger.h \ commands/trigger.h \ executor/tuptable.h \ diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl old mode 100644 new mode 100755 index 2f0a59bc87..bd397f45ac --- a/src/backend/nodes/gen_node_support.pl +++ b/src/backend/nodes/gen_node_support.pl @@ -61,6 +61,7 @@ my @all_input_files = qw( access/sdir.h access/tableam.h access/tsmapi.h + commands/copyapi.h commands/event_trigger.h commands/trigger.h executor/tuptable.h @@ -85,6 +86,7 @@ my @nodetag_only_files = qw( access/sdir.h access/tableam.h access/tsmapi.h + commands/copyapi.h commands/event_trigger.h commands/trigger.h executor/tuptable.h diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index a3a991f634..d308780c43 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -373,6 +373,7 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(fdw_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(table_am_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(index_am_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(tsm_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(copy_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(internal); PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement); PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 29af4ce65d..d4e426687c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -7617,6 +7617,12 @@ { oid => '3312', descr => 'I/O', proname => 'tsm_handler_out', prorettype => 'cstring', proargtypes => 'tsm_handler', prosrc => 'tsm_handler_out' }, +{ oid => '8753', descr => 'I/O', + proname => 'copy_handler_in', proisstrict => 'f', prorettype => 'copy_handler', + proargtypes => 'cstring', prosrc => 'copy_handler_in' }, +{ oid => '8754', descr => 'I/O', + proname => 'copy_handler_out', prorettype => 'cstring', + proargtypes => 'copy_handler', prosrc => 'copy_handler_out' }, { oid => '267', descr => 'I/O', proname => 'table_am_handler_in', proisstrict => 'f', prorettype => 'table_am_handler', proargtypes => 'cstring', diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat index d29194da31..2040d5da83 100644 --- a/src/include/catalog/pg_type.dat +++ b/src/include/catalog/pg_type.dat @@ -632,6 +632,12 @@ typcategory => 'P', typinput => 'tsm_handler_in', typoutput => 'tsm_handler_out', typreceive => '-', typsend => '-', typalign => 'i' }, +{ oid => '8752', + descr => 'pseudo-type for the result of a copy to/from method functoin', + typname => 'copy_handler', typlen => '4', typbyval => 't', typtype => 'p', + typcategory => 'P', typinput => 'copy_handler_in', + typoutput => 'copy_handler_out', typreceive => '-', typsend => '-', + typalign => 'i' }, { oid => '269', typname => 'table_am_handler', descr => 'pseudo-type for the result of a table AM handler function', diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index eb68f2fb7b..9c25e1c415 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -29,6 +29,8 @@ typedef void (*CopyToEnd_function) (CopyToState cstate); /* Routines for a COPY TO format implementation. */ typedef struct CopyToRoutine { + NodeTag type; + /* * Called for processing one COPY TO option. This will return false when * the given option is invalid. diff --git a/src/include/nodes/meson.build b/src/include/nodes/meson.build index b665e55b65..103df1a787 100644 --- a/src/include/nodes/meson.build +++ b/src/include/nodes/meson.build @@ -11,6 +11,7 @@ node_support_input_i = [ 'access/sdir.h', 'access/tableam.h', 'access/tsmapi.h', + 'commands/copyapi.h', 'commands/event_trigger.h', 'commands/trigger.h', 'executor/tuptable.h', diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index e32c8925f6..9d57b868d5 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -15,6 +15,7 @@ SUBDIRS = \ spgist_name_ops \ test_bloomfilter \ test_copy_callbacks \ + test_copy_format \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 397e0906e6..d76f2a6003 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -13,6 +13,7 @@ subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_bloomfilter') subdir('test_copy_callbacks') +subdir('test_copy_format') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') diff --git a/src/test/modules/test_copy_format/.gitignore b/src/test/modules/test_copy_format/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_copy_format/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_copy_format/Makefile b/src/test/modules/test_copy_format/Makefile new file mode 100644 index 0000000000..8497f91624 --- /dev/null +++ b/src/test/modules/test_copy_format/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_copy_format/Makefile + +MODULE_big = test_copy_format +OBJS = \ + $(WIN32RES) \ + test_copy_format.o +PGFILEDESC = "test_copy_format - test custom COPY FORMAT" + +EXTENSION = test_copy_format +DATA = test_copy_format--1.0.sql + +REGRESS = test_copy_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_copy_format +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out new file mode 100644 index 0000000000..3a24ae7b97 --- /dev/null +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -0,0 +1,17 @@ +CREATE EXTENSION test_copy_format; +CREATE TABLE public.test (a INT, b INT, c INT); +INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test TO stdout WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +NOTICE: test_copy_format: is_from=false +NOTICE: CopyToProcessOption: "option_before"="before" +NOTICE: CopyToProcessOption: "option_after"="after" +NOTICE: CopyToGetFormat +NOTICE: CopyToStart: natts=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToEnd diff --git a/src/test/modules/test_copy_format/meson.build b/src/test/modules/test_copy_format/meson.build new file mode 100644 index 0000000000..4cefe7b709 --- /dev/null +++ b/src/test/modules/test_copy_format/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_copy_format_sources = files( + 'test_copy_format.c', +) + +if host_system == 'windows' + test_copy_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_copy_format', + '--FILEDESC', 'test_copy_format - test custom COPY FORMAT',]) +endif + +test_copy_format = shared_module('test_copy_format', + test_copy_format_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_copy_format + +test_install_data += files( + 'test_copy_format.control', + 'test_copy_format--1.0.sql', +) + +tests += { + 'name': 'test_copy_format', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_copy_format', + ], + }, +} diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql new file mode 100644 index 0000000000..0eb7ed2e11 --- /dev/null +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -0,0 +1,8 @@ +CREATE EXTENSION test_copy_format; +CREATE TABLE public.test (a INT, b INT, c INT); +INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test TO stdout WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); diff --git a/src/test/modules/test_copy_format/test_copy_format--1.0.sql b/src/test/modules/test_copy_format/test_copy_format--1.0.sql new file mode 100644 index 0000000000..d24ea03ce9 --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format--1.0.sql @@ -0,0 +1,8 @@ +/* src/test/modules/test_copy_format/test_copy_format--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_copy_format" to load this file. \quit + +CREATE FUNCTION test_copy_format(internal) + RETURNS copy_handler + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c new file mode 100644 index 0000000000..a2219afcde --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -0,0 +1,77 @@ +/*-------------------------------------------------------------------------- + * + * test_copy_format.c + * Code for testing custom COPY format. + * + * Portions Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_copy_format/test_copy_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/defrem.h" + +PG_MODULE_MAGIC; + +static bool +CopyToProcessOption(CopyToState cstate, DefElem *defel) +{ + ereport(NOTICE, + (errmsg("CopyToProcessOption: \"%s\"=\"%s\"", + defel->defname, defGetString(defel)))); + return true; +} + +static int16 +CopyToGetFormat(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToGetFormat"))); + return 0; +} + +static void +CopyToStart(CopyToState cstate, TupleDesc tupDesc) +{ + ereport(NOTICE, (errmsg("CopyToStart: natts=%d", tupDesc->natts))); +} + +static void +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", slot->tts_nvalid))); +} + +static void +CopyToEnd(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToEnd"))); +} + +static const CopyToRoutine CopyToRoutineTestCopyFormat = { + .type = T_CopyToRoutine, + .CopyToProcessOption = CopyToProcessOption, + .CopyToGetFormat = CopyToGetFormat, + .CopyToStart = CopyToStart, + .CopyToOneRow = CopyToOneRow, + .CopyToEnd = CopyToEnd, +}; + +PG_FUNCTION_INFO_V1(test_copy_format); +Datum +test_copy_format(PG_FUNCTION_ARGS) +{ + bool is_from = PG_GETARG_BOOL(0); + + ereport(NOTICE, + (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false"))); + + if (is_from) + elog(ERROR, "COPY FROM isn't supported yet"); + + PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); +} diff --git a/src/test/modules/test_copy_format/test_copy_format.control b/src/test/modules/test_copy_format/test_copy_format.control new file mode 100644 index 0000000000..f05a636235 --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format.control @@ -0,0 +1,4 @@ +comment = 'Test code for custom COPY format' +default_version = '1.0' +module_pathname = '$libdir/test_copy_format' +relocatable = true -- 2.41.0 [application/octet-stream] v8-0003-Export-CopyToStateData.patch (13.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/5-v8-0003-Export-CopyToStateData.patch) download | inline diff: From c3a59753b1157dc8e47e719263f58677acc33178 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 14:54:10 +0900 Subject: [PATCH v8 03/10] Export CopyToStateData It's for custom COPY TO format handlers implemented as extension. This just moves codes. This doesn't change codes except CopyDest enum values. CopyDest enum values such as COPY_FILE are conflicted CopySource enum values defined in copyfrom_internal.h. So COPY_DEST_ prefix instead of COPY_ prefix is used. For example, COPY_FILE is renamed to COPY_DEST_FILE. Note that this change isn't enough to implement a custom COPY TO format handler as extension. We'll do the followings in a subsequent commit: 1. Add an opaque space for custom COPY TO format handler 2. Export CopySendEndOfRow() to flush buffer --- src/backend/commands/copyto.c | 74 +++----------------- src/include/commands/copy.h | 59 ---------------- src/include/commands/copyapi.h | 120 ++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 126 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6547b7c654..cfc74ee7b1 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -43,64 +43,6 @@ #include "utils/rel.h" #include "utils/snapmgr.h" -/* - * Represents the different dest cases we need to worry about at - * the bottom level - */ -typedef enum CopyDest -{ - COPY_FILE, /* to file (or a piped program) */ - COPY_FRONTEND, /* to frontend */ - COPY_CALLBACK, /* to callback function */ -} CopyDest; - -/* - * This struct contains all the state variables used throughout a COPY TO - * operation. - * - * Multi-byte encodings: all supported client-side encodings encode multi-byte - * characters by having the first byte's high bit set. Subsequent bytes of the - * character can have the high bit not set. When scanning data in such an - * encoding to look for a match to a single-byte (ie ASCII) character, we must - * use the full pg_encoding_mblen() machinery to skip over multibyte - * characters, else we might find a false match to a trailing byte. In - * supported server encodings, there is no possibility of a false match, and - * it's faster to make useless comparisons to trailing bytes than it is to - * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true - * when we have to do it the hard way. - */ -typedef struct CopyToStateData -{ - /* low-level state data */ - CopyDest copy_dest; /* type of copy source/destination */ - FILE *copy_file; /* used if copy_dest == COPY_FILE */ - StringInfo fe_msgbuf; /* used for all dests during COPY TO */ - - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - FmgrInfo *out_functions; /* lookup info for output functions */ - MemoryContext rowcontext; /* per-row evaluation context */ - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; - /* DestReceiver for COPY (query) TO */ typedef struct { @@ -160,7 +102,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) { switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: /* Default line termination depends on platform */ #ifndef WIN32 CopySendChar(cstate, '\n'); @@ -168,7 +110,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) CopySendString(cstate, "\r\n"); #endif break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); break; @@ -419,7 +361,7 @@ SendCopyBegin(CopyToState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->copy_dest = COPY_DEST_FRONTEND; } static void @@ -466,7 +408,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || ferror(cstate->copy_file)) @@ -500,11 +442,11 @@ CopySendEndOfRow(CopyToState cstate) errmsg("could not write to COPY file: %m"))); } break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; - case COPY_CALLBACK: + case COPY_DEST_CALLBACK: cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len); break; } @@ -877,12 +819,12 @@ BeginCopyTo(ParseState *pstate, /* See Multibyte encoding comment above */ cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - cstate->copy_dest = COPY_FILE; /* default */ + cstate->copy_dest = COPY_DEST_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->copy_dest = COPY_DEST_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 34bea880ca..b3f4682f95 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,69 +20,10 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -/* - * Represents whether a header line should be present, and whether it must - * match the actual names (which implies "true"). - */ -typedef enum CopyHeaderChoice -{ - COPY_HEADER_FALSE = 0, - COPY_HEADER_TRUE, - COPY_HEADER_MATCH, -} CopyHeaderChoice; - -/* - * Represents where to save input processing errors. More values to be added - * in the future. - */ -typedef enum CopyOnErrorChoice -{ - COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */ - COPY_ON_ERROR_IGNORE, /* ignore errors */ -} CopyOnErrorChoice; - -/* - * A struct to hold COPY options, in a parsed form. All of these are related - * to formatting, except for 'freeze', which doesn't really belong here, but - * it's expedient to parse it along with all the other options. - */ -typedef struct CopyFormatOptions -{ - /* parameters from the COPY command */ - int file_encoding; /* file or remote side's character encoding, - * -1 if not specified */ - bool binary; /* binary format? */ - bool freeze; /* freeze rows on loading? */ - bool csv_mode; /* Comma Separated Value format? */ - CopyHeaderChoice header_line; /* header line? */ - char *null_print; /* NULL marker string (server encoding!) */ - int null_print_len; /* length of same */ - char *null_print_client; /* same converted to file encoding */ - char *default_print; /* DEFAULT marker string */ - int default_print_len; /* length of same */ - char *delim; /* column delimiter (must be 1 byte) */ - char *quote; /* CSV quote char (must be 1 byte) */ - char *escape; /* CSV escape char (must be 1 byte) */ - List *force_quote; /* list of column names */ - bool force_quote_all; /* FORCE_QUOTE *? */ - bool *force_quote_flags; /* per-column CSV FQ flags */ - List *force_notnull; /* list of column names */ - bool force_notnull_all; /* FORCE_NOT_NULL *? */ - bool *force_notnull_flags; /* per-column CSV FNN flags */ - List *force_null; /* list of column names */ - bool force_null_all; /* FORCE_NULL *? */ - bool *force_null_flags; /* per-column CSV FN flags */ - bool convert_selectively; /* do selective binary conversion? */ - CopyOnErrorChoice on_error; /* what to do when error happened */ - List *convert_select; /* list of column names (can be NIL) */ - CopyToRoutine *to_routine; /* callback routines for COPY TO */ -} CopyFormatOptions; - /* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); -typedef void (*copy_data_dest_cb) (void *data, int len); extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 9c25e1c415..a869d78d72 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,10 +14,10 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "executor/execdesc.h" #include "executor/tuptable.h" #include "nodes/parsenodes.h" -/* This is private in commands/copyto.c */ typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); @@ -58,4 +58,122 @@ extern CopyToRoutine CopyToRoutineText; extern CopyToRoutine CopyToRoutineCSV; extern CopyToRoutine CopyToRoutineBinary; +/* + * Represents whether a header line should be present, and whether it must + * match the actual names (which implies "true"). + */ +typedef enum CopyHeaderChoice +{ + COPY_HEADER_FALSE = 0, + COPY_HEADER_TRUE, + COPY_HEADER_MATCH, +} CopyHeaderChoice; + +/* + * Represents where to save input processing errors. More values to be added + * in the future. + */ +typedef enum CopyOnErrorChoice +{ + COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */ + COPY_ON_ERROR_IGNORE, /* ignore errors */ +} CopyOnErrorChoice; + +/* + * A struct to hold COPY options, in a parsed form. All of these are related + * to formatting, except for 'freeze', which doesn't really belong here, but + * it's expedient to parse it along with all the other options. + */ +typedef struct CopyFormatOptions +{ + /* parameters from the COPY command */ + int file_encoding; /* file or remote side's character encoding, + * -1 if not specified */ + bool binary; /* binary format? */ + bool freeze; /* freeze rows on loading? */ + bool csv_mode; /* Comma Separated Value format? */ + CopyHeaderChoice header_line; /* header line? */ + char *null_print; /* NULL marker string (server encoding!) */ + int null_print_len; /* length of same */ + char *null_print_client; /* same converted to file encoding */ + char *default_print; /* DEFAULT marker string */ + int default_print_len; /* length of same */ + char *delim; /* column delimiter (must be 1 byte) */ + char *quote; /* CSV quote char (must be 1 byte) */ + char *escape; /* CSV escape char (must be 1 byte) */ + List *force_quote; /* list of column names */ + bool force_quote_all; /* FORCE_QUOTE *? */ + bool *force_quote_flags; /* per-column CSV FQ flags */ + List *force_notnull; /* list of column names */ + bool force_notnull_all; /* FORCE_NOT_NULL *? */ + bool *force_notnull_flags; /* per-column CSV FNN flags */ + List *force_null; /* list of column names */ + bool force_null_all; /* FORCE_NULL *? */ + bool *force_null_flags; /* per-column CSV FN flags */ + bool convert_selectively; /* do selective binary conversion? */ + CopyOnErrorChoice on_error; /* what to do when error happened */ + List *convert_select; /* list of column names (can be NIL) */ + CopyToRoutine *to_routine; /* callback routines for COPY TO */ +} CopyFormatOptions; + +/* + * Represents the different dest cases we need to worry about at + * the bottom level + */ +typedef enum CopyDest +{ + COPY_DEST_FILE, /* to file (or a piped program) */ + COPY_DEST_FRONTEND, /* to frontend */ + COPY_DEST_CALLBACK, /* to callback function */ +} CopyDest; + +typedef void (*copy_data_dest_cb) (void *data, int len); + +/* + * This struct contains all the state variables used throughout a COPY TO + * operation. + * + * Multi-byte encodings: all supported client-side encodings encode multi-byte + * characters by having the first byte's high bit set. Subsequent bytes of the + * character can have the high bit not set. When scanning data in such an + * encoding to look for a match to a single-byte (ie ASCII) character, we must + * use the full pg_encoding_mblen() machinery to skip over multibyte + * characters, else we might find a false match to a trailing byte. In + * supported server encodings, there is no possibility of a false match, and + * it's faster to make useless comparisons to trailing bytes than it is to + * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true + * when we have to do it the hard way. + */ +typedef struct CopyToStateData +{ + /* low-level state data */ + CopyDest copy_dest; /* type of copy source/destination */ + FILE *copy_file; /* used if copy_dest == COPY_FILE */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *out_functions; /* lookup info for output functions */ + MemoryContext rowcontext; /* per-row evaluation context */ + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyToStateData; + #endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0005-Extract-COPY-FROM-format-implementations.patch (24.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/6-v8-0005-Extract-COPY-FROM-format-implementations.patch) download | inline diff: From 781955f19ad27cdd66748be539bf45cf1b925856 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 17:21:23 +0900 Subject: [PATCH v8 05/10] Extract COPY FROM format implementations This doesn't change the current behavior. This just introduces CopyFromRoutine, which just has function pointers of format implementation like TupleTableSlotOps, and use it for existing "text", "csv" and "binary" format implementations. Note that CopyFromRoutine can't be used from extensions yet because CopyRead*() aren't exported yet. Extensions can't read data from a source without CopyRead*(). They will be exported by subsequent patches. --- src/backend/commands/copy.c | 3 + src/backend/commands/copyfrom.c | 216 +++++++++++---- src/backend/commands/copyfromparse.c | 326 ++++++++++++----------- src/include/commands/copy.h | 3 - src/include/commands/copyapi.h | 44 +++ src/include/commands/copyfrom_internal.h | 4 + 6 files changed, 391 insertions(+), 205 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 6f0db0ae7c..ec6dfff8ab 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -459,12 +459,14 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, else if (strcmp(format, "csv") == 0) { opts_out->csv_mode = true; + opts_out->from_routine = &CopyFromRoutineCSV; opts_out->to_routine = &CopyToRoutineCSV; return; } else if (strcmp(format, "binary") == 0) { opts_out->binary = true; + opts_out->from_routine = &CopyFromRoutineBinary; opts_out->to_routine = &CopyToRoutineBinary; return; } @@ -533,6 +535,7 @@ ProcessCopyOptions(ParseState *pstate, opts_out->file_encoding = -1; /* Text is the default format. */ + opts_out->from_routine = &CopyFromRoutineText; opts_out->to_routine = &CopyToRoutineText; /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index fb3d4d9296..d556ebb5d6 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -108,6 +108,170 @@ static char *limit_printout_length(const char *str); static void ClosePipeFromProgram(CopyFromState cstate); + +/* + * CopyFromRoutine implementations. + */ + +/* + * CopyFromRoutine implementation for "text" and "csv". CopyFromText*() + * refer cstate->opts.csv_mode and change their behavior. We can split this + * implementation and stop referring cstate->opts.csv_mode later. + */ + +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ +static bool +CopyFromTextProcessOption(CopyFromState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyFromTextGetFormat(CopyFromState cstate) +{ + return 0; +} + +static void +CopyFromTextStart(CopyFromState cstate, TupleDesc tupDesc) +{ + AttrNumber num_phys_attrs = tupDesc->natts; + AttrNumber attr_count; + + /* + * If encoding conversion is needed, we need another buffer to hold the + * converted input data. Otherwise, we can just point input_buf to the + * same buffer as raw_buf. + */ + if (cstate->need_transcoding) + { + cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); + cstate->input_buf_index = cstate->input_buf_len = 0; + } + else + cstate->input_buf = cstate->raw_buf; + cstate->input_reached_eof = false; + + initStringInfo(&cstate->line_buf); + + /* + * Pick up the required catalog information for each attribute in the + * relation, including the input function, the element type (to pass to + * the input function). + */ + cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); + for (int attnum = 1; attnum <= num_phys_attrs; attnum++) + { + Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1); + Oid in_func_oid; + + /* We don't need info for dropped attributes */ + if (att->attisdropped) + continue; + + /* Fetch the input function and typioparam info */ + getTypeInputInfo(att->atttypid, + &in_func_oid, &cstate->typioparams[attnum - 1]); + fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]); + } + + /* create workspace for CopyReadAttributes results */ + attr_count = list_length(cstate->attnumlist); + cstate->max_fields = attr_count; + cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); +} + +static void +CopyFromTextEnd(CopyFromState cstate) +{ +} + +/* + * CopyFromRoutine implementation for "binary". + */ + +/* All "binary" options are parsed in ProcessCopyOptions(). We may move the + * code to here later. */ +static bool +CopyFromBinaryProcessOption(CopyFromState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyFromBinaryGetFormat(CopyFromState cstate) +{ + return 1; +} + +static void +CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc) +{ + AttrNumber num_phys_attrs = tupDesc->natts; + + /* + * Pick up the required catalog information for each attribute in the + * relation, including the input function, the element type (to pass to + * the input function). + */ + cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); + for (int attnum = 1; attnum <= num_phys_attrs; attnum++) + { + Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1); + Oid in_func_oid; + + /* We don't need info for dropped attributes */ + if (att->attisdropped) + continue; + + /* Fetch the input function and typioparam info */ + getTypeBinaryInputInfo(att->atttypid, + &in_func_oid, &cstate->typioparams[attnum - 1]); + fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]); + } + + /* Read and verify binary header */ + ReceiveCopyBinaryHeader(cstate); +} + +static void +CopyFromBinaryEnd(CopyFromState cstate) +{ +} + +CopyFromRoutine CopyFromRoutineText = { + .CopyFromProcessOption = CopyFromTextProcessOption, + .CopyFromGetFormat = CopyFromTextGetFormat, + .CopyFromStart = CopyFromTextStart, + .CopyFromOneRow = CopyFromTextOneRow, + .CopyFromEnd = CopyFromTextEnd, +}; + +/* + * We can use the same CopyFromRoutine for both of "text" and "csv" because + * CopyFromText*() refer cstate->opts.csv_mode and change their behavior. We can + * split the implementations and stop referring cstate->opts.csv_mode later. + */ +CopyFromRoutine CopyFromRoutineCSV = { + .CopyFromProcessOption = CopyFromTextProcessOption, + .CopyFromGetFormat = CopyFromTextGetFormat, + .CopyFromStart = CopyFromTextStart, + .CopyFromOneRow = CopyFromTextOneRow, + .CopyFromEnd = CopyFromTextEnd, +}; + +CopyFromRoutine CopyFromRoutineBinary = { + .CopyFromProcessOption = CopyFromBinaryProcessOption, + .CopyFromGetFormat = CopyFromBinaryGetFormat, + .CopyFromStart = CopyFromBinaryStart, + .CopyFromOneRow = CopyFromBinaryOneRow, + .CopyFromEnd = CopyFromBinaryEnd, +}; + + /* * error context callback for COPY FROM * @@ -1384,9 +1548,6 @@ BeginCopyFrom(ParseState *pstate, TupleDesc tupDesc; AttrNumber num_phys_attrs, num_defaults; - FmgrInfo *in_functions; - Oid *typioparams; - Oid in_func_oid; int *defmap; ExprState **defexprs; MemoryContext oldcontext; @@ -1571,25 +1732,6 @@ BeginCopyFrom(ParseState *pstate, cstate->raw_buf_index = cstate->raw_buf_len = 0; cstate->raw_reached_eof = false; - if (!cstate->opts.binary) - { - /* - * If encoding conversion is needed, we need another buffer to hold - * the converted input data. Otherwise, we can just point input_buf - * to the same buffer as raw_buf. - */ - if (cstate->need_transcoding) - { - cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); - cstate->input_buf_index = cstate->input_buf_len = 0; - } - else - cstate->input_buf = cstate->raw_buf; - cstate->input_reached_eof = false; - - initStringInfo(&cstate->line_buf); - } - initStringInfo(&cstate->attribute_buf); /* Assign range table and rteperminfos, we'll need them in CopyFrom. */ @@ -1608,8 +1750,6 @@ BeginCopyFrom(ParseState *pstate, * the input function), and info about defaults and constraints. (Which * input function we use depends on text/binary format choice.) */ - in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); - typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); defmap = (int *) palloc(num_phys_attrs * sizeof(int)); defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *)); @@ -1621,15 +1761,6 @@ BeginCopyFrom(ParseState *pstate, if (att->attisdropped) continue; - /* Fetch the input function and typioparam info */ - if (cstate->opts.binary) - getTypeBinaryInputInfo(att->atttypid, - &in_func_oid, &typioparams[attnum - 1]); - else - getTypeInputInfo(att->atttypid, - &in_func_oid, &typioparams[attnum - 1]); - fmgr_info(in_func_oid, &in_functions[attnum - 1]); - /* Get default info if available */ defexprs[attnum - 1] = NULL; @@ -1689,8 +1820,6 @@ BeginCopyFrom(ParseState *pstate, cstate->bytes_processed = 0; /* We keep those variables in cstate. */ - cstate->in_functions = in_functions; - cstate->typioparams = typioparams; cstate->defmap = defmap; cstate->defexprs = defexprs; cstate->volatile_defexprs = volatile_defexprs; @@ -1763,20 +1892,7 @@ BeginCopyFrom(ParseState *pstate, pgstat_progress_update_multi_param(3, progress_cols, progress_vals); - if (cstate->opts.binary) - { - /* Read and verify binary header */ - ReceiveCopyBinaryHeader(cstate); - } - - /* create workspace for CopyReadAttributes results */ - if (!cstate->opts.binary) - { - AttrNumber attr_count = list_length(cstate->attnumlist); - - cstate->max_fields = attr_count; - cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); - } + cstate->opts.from_routine->CopyFromStart(cstate, tupDesc); MemoryContextSwitchTo(oldcontext); @@ -1789,6 +1905,8 @@ BeginCopyFrom(ParseState *pstate, void EndCopyFrom(CopyFromState cstate) { + cstate->opts.from_routine->CopyFromEnd(cstate); + /* No COPY FROM related resources except memory. */ if (cstate->is_program) { diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 7cacd0b752..49632f75e4 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -172,7 +172,7 @@ ReceiveCopyBegin(CopyFromState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = cstate->opts.from_routine->CopyFromGetFormat(cstate); int i; pq_beginmessage(&buf, PqMsg_CopyInResponse); @@ -840,199 +840,219 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) return true; } -/* - * Read next tuple from file for COPY FROM. Return false if no more tuples. - * - * 'econtext' is used to evaluate default expression for each column that is - * either not read from the file or is using the DEFAULT option of COPY FROM. - * It can be NULL when no default values are used, i.e. when all columns are - * read from the file, and DEFAULT option is unset. - * - * 'values' and 'nulls' arrays must be the same length as columns of the - * relation passed to BeginCopyFrom. This function fills the arrays. - */ bool -NextCopyFrom(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls) +CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { TupleDesc tupDesc; - AttrNumber num_phys_attrs, - attr_count, - num_defaults = cstate->num_defaults; + AttrNumber attr_count; FmgrInfo *in_functions = cstate->in_functions; Oid *typioparams = cstate->typioparams; - int i; - int *defmap = cstate->defmap; ExprState **defexprs = cstate->defexprs; + char **field_strings; + ListCell *cur; + int fldct; + int fieldno; + char *string; tupDesc = RelationGetDescr(cstate->rel); - num_phys_attrs = tupDesc->natts; attr_count = list_length(cstate->attnumlist); - /* Initialize all values for row to NULL */ - MemSet(values, 0, num_phys_attrs * sizeof(Datum)); - MemSet(nulls, true, num_phys_attrs * sizeof(bool)); - MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); + /* read raw fields in the next line */ + if (!NextCopyFromRawFields(cstate, &field_strings, &fldct)) + return false; - if (!cstate->opts.binary) - { - char **field_strings; - ListCell *cur; - int fldct; - int fieldno; - char *string; + /* check for overflowing fields */ + if (attr_count > 0 && fldct > attr_count) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("extra data after last expected column"))); - /* read raw fields in the next line */ - if (!NextCopyFromRawFields(cstate, &field_strings, &fldct)) - return false; + fieldno = 0; - /* check for overflowing fields */ - if (attr_count > 0 && fldct > attr_count) + /* Loop to read the user attributes on the line. */ + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (fieldno >= fldct) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("extra data after last expected column"))); + errmsg("missing data for column \"%s\"", + NameStr(att->attname)))); + string = field_strings[fieldno++]; - fieldno = 0; - - /* Loop to read the user attributes on the line. */ - foreach(cur, cstate->attnumlist) + if (cstate->convert_select_flags && + !cstate->convert_select_flags[m]) { - int attnum = lfirst_int(cur); - int m = attnum - 1; - Form_pg_attribute att = TupleDescAttr(tupDesc, m); - - if (fieldno >= fldct) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("missing data for column \"%s\"", - NameStr(att->attname)))); - string = field_strings[fieldno++]; - - if (cstate->convert_select_flags && - !cstate->convert_select_flags[m]) - { - /* ignore input field, leaving column as NULL */ - continue; - } + /* ignore input field, leaving column as NULL */ + continue; + } - if (cstate->opts.csv_mode) + if (cstate->opts.csv_mode) + { + if (string == NULL && + cstate->opts.force_notnull_flags[m]) { - if (string == NULL && - cstate->opts.force_notnull_flags[m]) - { - /* - * FORCE_NOT_NULL option is set and column is NULL - - * convert it to the NULL string. - */ - string = cstate->opts.null_print; - } - else if (string != NULL && cstate->opts.force_null_flags[m] - && strcmp(string, cstate->opts.null_print) == 0) - { - /* - * FORCE_NULL option is set and column matches the NULL - * string. It must have been quoted, or otherwise the - * string would already have been set to NULL. Convert it - * to NULL as specified. - */ - string = NULL; - } + /* + * FORCE_NOT_NULL option is set and column is NULL - convert + * it to the NULL string. + */ + string = cstate->opts.null_print; } - - cstate->cur_attname = NameStr(att->attname); - cstate->cur_attval = string; - - if (string != NULL) - nulls[m] = false; - - if (cstate->defaults[m]) + else if (string != NULL && cstate->opts.force_null_flags[m] + && strcmp(string, cstate->opts.null_print) == 0) { /* - * The caller must supply econtext and have switched into the - * per-tuple memory context in it. + * FORCE_NULL option is set and column matches the NULL + * string. It must have been quoted, or otherwise the string + * would already have been set to NULL. Convert it to NULL as + * specified. */ - Assert(econtext != NULL); - Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); - - values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); + string = NULL; } + } + + cstate->cur_attname = NameStr(att->attname); + cstate->cur_attval = string; + + if (string != NULL) + nulls[m] = false; + if (cstate->defaults[m]) + { /* - * If ON_ERROR is specified with IGNORE, skip rows with soft - * errors + * The caller must supply econtext and have switched into the + * per-tuple memory context in it. */ - else if (!InputFunctionCallSafe(&in_functions[m], - string, - typioparams[m], - att->atttypmod, - (Node *) cstate->escontext, - &values[m])) - { - cstate->num_errors++; - return true; - } + Assert(econtext != NULL); + Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; + values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); } - Assert(fieldno == attr_count); + /* + * If ON_ERROR is specified with IGNORE, skip rows with soft errors + */ + else if (!InputFunctionCallSafe(&in_functions[m], + string, + typioparams[m], + att->atttypmod, + (Node *) cstate->escontext, + &values[m])) + { + cstate->num_errors++; + return true; + } + + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; } - else - { - /* binary */ - int16 fld_count; - ListCell *cur; - cstate->cur_lineno++; + Assert(fieldno == attr_count); - if (!CopyGetInt16(cstate, &fld_count)) - { - /* EOF detected (end of file, or protocol-level EOF) */ - return false; - } + return true; +} - if (fld_count == -1) - { - /* - * Received EOF marker. Wait for the protocol-level EOF, and - * complain if it doesn't come immediately. In COPY FROM STDIN, - * this ensures that we correctly handle CopyFail, if client - * chooses to send that now. When copying from file, we could - * ignore the rest of the file like in text mode, but we choose to - * be consistent with the COPY FROM STDIN case. - */ - char dummy; +bool +CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + TupleDesc tupDesc; + AttrNumber attr_count; + FmgrInfo *in_functions = cstate->in_functions; + Oid *typioparams = cstate->typioparams; + int16 fld_count; + ListCell *cur; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("received copy data after EOF marker"))); - return false; - } + tupDesc = RelationGetDescr(cstate->rel); + attr_count = list_length(cstate->attnumlist); - if (fld_count != attr_count) + cstate->cur_lineno++; + + if (!CopyGetInt16(cstate, &fld_count)) + { + /* EOF detected (end of file, or protocol-level EOF) */ + return false; + } + + if (fld_count == -1) + { + /* + * Received EOF marker. Wait for the protocol-level EOF, and complain + * if it doesn't come immediately. In COPY FROM STDIN, this ensures + * that we correctly handle CopyFail, if client chooses to send that + * now. When copying from file, we could ignore the rest of the file + * like in text mode, but we choose to be consistent with the COPY + * FROM STDIN case. + */ + char dummy; + + if (CopyReadBinaryData(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("row field count is %d, expected %d", - (int) fld_count, attr_count))); + errmsg("received copy data after EOF marker"))); + return false; + } - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - int m = attnum - 1; - Form_pg_attribute att = TupleDescAttr(tupDesc, m); - - cstate->cur_attname = NameStr(att->attname); - values[m] = CopyReadBinaryAttribute(cstate, - &in_functions[m], - typioparams[m], - att->atttypmod, - &nulls[m]); - cstate->cur_attname = NULL; - } + if (fld_count != attr_count) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("row field count is %d, expected %d", + (int) fld_count, attr_count))); + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + cstate->cur_attname = NameStr(att->attname); + values[m] = CopyReadBinaryAttribute(cstate, + &in_functions[m], + typioparams[m], + att->atttypmod, + &nulls[m]); + cstate->cur_attname = NULL; } + return true; +} + +/* + * Read next tuple from file for COPY FROM. Return false if no more tuples. + * + * 'econtext' is used to evaluate default expression for each column that is + * either not read from the file or is using the DEFAULT option of COPY FROM. + * It can be NULL when no default values are used, i.e. when all columns are + * read from the file, and DEFAULT option is unset. + * + * 'values' and 'nulls' arrays must be the same length as columns of the + * relation passed to BeginCopyFrom. This function fills the arrays. + */ +bool +NextCopyFrom(CopyFromState cstate, ExprContext *econtext, + Datum *values, bool *nulls) +{ + TupleDesc tupDesc; + AttrNumber num_phys_attrs, + num_defaults = cstate->num_defaults; + int i; + int *defmap = cstate->defmap; + ExprState **defexprs = cstate->defexprs; + + tupDesc = RelationGetDescr(cstate->rel); + num_phys_attrs = tupDesc->natts; + + /* Initialize all values for row to NULL */ + MemSet(values, 0, num_phys_attrs * sizeof(Datum)); + MemSet(nulls, true, num_phys_attrs * sizeof(bool)); + MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); + + if (!cstate->opts.from_routine->CopyFromOneRow(cstate, econtext, values, + nulls)) + return false; + /* * Now compute and insert any defaults available for the columns not * provided by the input data. Anything not processed here or above will diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index b3f4682f95..df29d42555 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,9 +20,6 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -/* This is private in commands/copyfrom.c */ -typedef struct CopyFromStateData *CopyFromState; - typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index ffad433a21..323e4705d2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -18,6 +18,49 @@ #include "executor/tuptable.h" #include "nodes/parsenodes.h" +/* This is private in commands/copyfrom.c */ +typedef struct CopyFromStateData *CopyFromState; + +typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); +typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate); +typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc); +typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); +typedef void (*CopyFromEnd_function) (CopyFromState cstate); + +/* Routines for a COPY FROM format implementation. */ +typedef struct CopyFromRoutine +{ + /* + * Called for processing one COPY FROM option. This will return false when + * the given option is invalid. + */ + CopyFromProcessOption_function CopyFromProcessOption; + + /* + * Called when COPY FROM is started. This will return a format as int16 + * value. It's used for the CopyInResponse message. + */ + CopyFromGetFormat_function CopyFromGetFormat; + + /* + * Called when COPY FROM is started. This will initialize something and + * receive a header. + */ + CopyFromStart_function CopyFromStart; + + /* Copy one row. It returns false if no more tuples. */ + CopyFromOneRow_function CopyFromOneRow; + + /* Called when COPY FROM is ended. This will finalize something. */ + CopyFromEnd_function CopyFromEnd; +} CopyFromRoutine; + +/* Built-in CopyFromRoutine for "text", "csv" and "binary". */ +extern CopyFromRoutine CopyFromRoutineText; +extern CopyFromRoutine CopyFromRoutineCSV; +extern CopyFromRoutine CopyFromRoutineBinary; + + typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); @@ -113,6 +156,7 @@ typedef struct CopyFormatOptions bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ List *convert_select; /* list of column names (can be NIL) */ + CopyFromRoutine *from_routine; /* callback routines for COPY FROM */ CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index cad52fcc78..921c1513f7 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -183,4 +183,8 @@ typedef struct CopyFromStateData extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); +extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); +extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); + + #endif /* COPYFROM_INTERNAL_H */ -- 2.41.0 [application/octet-stream] v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch (7.6K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/7-v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch) download | inline diff: From f0a8151feff44823881c3c4e1e7aca4f9bd690d5 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Sat, 27 Jan 2024 09:53:31 +0800 Subject: [PATCH v8 09/10] change CopyToGetFormat to CopyToSendCopyBegin and export more api Signed-off-by: Zhao Junwang <[email protected]> --- src/backend/commands/copyto.c | 65 ++++++++++--------- src/include/commands/copyapi.h | 12 ++-- .../test_copy_format/test_copy_format.c | 7 +- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b5d8678394..e2a4964015 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -66,11 +66,6 @@ static void CopyAttributeOutCSV(CopyToState cstate, const char *string, /* Low-level communications functions */ static void SendCopyBegin(CopyToState cstate); static void SendCopyEnd(CopyToState cstate); -static void CopySendData(CopyToState cstate, const void *databuf, int datasize); -static void CopySendString(CopyToState cstate, const char *str); -static void CopySendChar(CopyToState cstate, char c); -static void CopySendInt32(CopyToState cstate, int32 val); -static void CopySendInt16(CopyToState cstate, int16 val); /* * CopyToRoutine implementations. @@ -90,10 +85,20 @@ CopyToTextProcessOption(CopyToState cstate, DefElem *defel) return false; } -static int16 -CopyToTextGetFormat(CopyToState cstate) +static void +CopyToTextSendCopyBegin(CopyToState cstate) { - return 0; + StringInfoData buf; + int natts = list_length(cstate->attnumlist); + int16 format = 0; + int i; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + pq_sendint16(&buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(&buf, format); /* per-column formats */ + pq_endmessage(&buf); } static void @@ -230,10 +235,20 @@ CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel) return false; } -static int16 -CopyToBinaryGetFormat(CopyToState cstate) +static void +CopyToBinarySendCopyBegin(CopyToState cstate) { - return 1; + StringInfoData buf; + int natts = list_length(cstate->attnumlist); + int16 format = 1; + int i; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + pq_sendint16(&buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(&buf, format); /* per-column formats */ + pq_endmessage(&buf); } static void @@ -315,7 +330,7 @@ CopyToBinaryEnd(CopyToState cstate) CopyToRoutine CopyToRoutineText = { .CopyToProcessOption = CopyToTextProcessOption, - .CopyToGetFormat = CopyToTextGetFormat, + .CopyToSendCopyBegin = CopyToTextSendCopyBegin, .CopyToStart = CopyToTextStart, .CopyToOneRow = CopyToTextOneRow, .CopyToEnd = CopyToTextEnd, @@ -328,7 +343,7 @@ CopyToRoutine CopyToRoutineText = { */ CopyToRoutine CopyToRoutineCSV = { .CopyToProcessOption = CopyToTextProcessOption, - .CopyToGetFormat = CopyToTextGetFormat, + .CopyToSendCopyBegin = CopyToTextSendCopyBegin, .CopyToStart = CopyToTextStart, .CopyToOneRow = CopyToTextOneRow, .CopyToEnd = CopyToTextEnd, @@ -336,7 +351,7 @@ CopyToRoutine CopyToRoutineCSV = { CopyToRoutine CopyToRoutineBinary = { .CopyToProcessOption = CopyToBinaryProcessOption, - .CopyToGetFormat = CopyToBinaryGetFormat, + .CopyToSendCopyBegin = CopyToBinarySendCopyBegin, .CopyToStart = CopyToBinaryStart, .CopyToOneRow = CopyToBinaryOneRow, .CopyToEnd = CopyToBinaryEnd, @@ -349,17 +364,7 @@ CopyToRoutine CopyToRoutineBinary = { static void SendCopyBegin(CopyToState cstate) { - StringInfoData buf; - int natts = list_length(cstate->attnumlist); - int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate); - int i; - - pq_beginmessage(&buf, PqMsg_CopyOutResponse); - pq_sendbyte(&buf, format); /* overall format */ - pq_sendint16(&buf, natts); - for (i = 0; i < natts; i++) - pq_sendint16(&buf, format); /* per-column formats */ - pq_endmessage(&buf); + cstate->opts.to_routine->CopyToSendCopyBegin(cstate); cstate->copy_dest = COPY_DEST_FRONTEND; } @@ -382,19 +387,19 @@ SendCopyEnd(CopyToState cstate) * NB: no data conversion is applied by these functions *---------- */ -static void +void CopySendData(CopyToState cstate, const void *databuf, int datasize) { appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize); } -static void +void CopySendString(CopyToState cstate, const char *str) { appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str)); } -static void +void CopySendChar(CopyToState cstate, char c) { appendStringInfoCharMacro(cstate->fe_msgbuf, c); @@ -464,7 +469,7 @@ CopyToStateFlush(CopyToState cstate) /* * CopySendInt32 sends an int32 in network byte order */ -static inline void +inline void CopySendInt32(CopyToState cstate, int32 val) { uint32 buf; @@ -476,7 +481,7 @@ CopySendInt32(CopyToState cstate, int32 val) /* * CopySendInt16 sends an int16 in network byte order */ -static inline void +inline void CopySendInt16(CopyToState cstate, int16 val) { uint16 buf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 22accc83ab..0a05b24c54 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -67,7 +67,7 @@ extern CopyFromRoutine CopyFromRoutineBinary; typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); -typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToSendCopyBegin_function) (CopyToState cstate); typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); typedef void (*CopyToEnd_function) (CopyToState cstate); @@ -84,10 +84,9 @@ typedef struct CopyToRoutine CopyToProcessOption_function CopyToProcessOption; /* - * Called when COPY TO is started. This will return a format as int16 - * value. It's used for the CopyOutResponse message. + * Called when COPY TO is started. */ - CopyToGetFormat_function CopyToGetFormat; + CopyToSendCopyBegin_function CopyToSendCopyBegin; /* Called when COPY TO is started. This will send a header. */ CopyToStart_function CopyToStart; @@ -384,6 +383,11 @@ typedef struct CopyToStateData void *opaque; /* private space */ } CopyToStateData; +extern void CopySendData(CopyToState cstate, const void *databuf, int datasize); +extern void CopySendString(CopyToState cstate, const char *str); +extern void CopySendChar(CopyToState cstate, char c); +extern void CopySendInt32(CopyToState cstate, int32 val); +extern void CopySendInt16(CopyToState cstate, int16 val); extern void CopyToStateFlush(CopyToState cstate); #endif /* COPYAPI_H */ diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 5e1b40e881..d833f22bbf 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -71,11 +71,10 @@ CopyToProcessOption(CopyToState cstate, DefElem *defel) return true; } -static int16 -CopyToGetFormat(CopyToState cstate) +static void +CopyToSendCopyBegin(CopyToState cstate) { ereport(NOTICE, (errmsg("CopyToGetFormat"))); - return 0; } static void @@ -99,7 +98,7 @@ CopyToEnd(CopyToState cstate) static const CopyToRoutine CopyToRoutineTestCopyFormat = { .type = T_CopyToRoutine, .CopyToProcessOption = CopyToProcessOption, - .CopyToGetFormat = CopyToGetFormat, + .CopyToSendCopyBegin = CopyToSendCopyBegin, .CopyToStart = CopyToStart, .CopyToOneRow = CopyToOneRow, .CopyToEnd = CopyToEnd, -- 2.41.0 [application/octet-stream] v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch (7.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/8-v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch) download | inline diff: From f48e7b629a8d15fc70cd4cc4737dd2ad61910cc9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 11:07:14 +0900 Subject: [PATCH v8 06/10] Add support for adding custom COPY FROM format We use the same approach as we used for custom COPY TO format. Now, custom COPY format handler can return COPY TO format routines or COPY FROM format routines based on the "is_from" argument: copy_handler(true) returns CopyToRoutine copy_handler(false) returns CopyFromRoutine --- src/backend/commands/copy.c | 53 +++++++++++++------ src/include/commands/copyapi.h | 2 + .../expected/test_copy_format.out | 12 +++++ .../test_copy_format/sql/test_copy_format.sql | 6 +++ .../test_copy_format/test_copy_format.c | 50 +++++++++++++++-- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index ec6dfff8ab..479f36868c 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -472,12 +472,9 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, } /* custom format */ - if (!is_from) - { - funcargtypes[0] = INTERNALOID; - handlerOid = LookupFuncName(list_make1(makeString(format)), 1, - funcargtypes, true); - } + funcargtypes[0] = INTERNALOID; + handlerOid = LookupFuncName(list_make1(makeString(format)), 1, + funcargtypes, true); if (!OidIsValid(handlerOid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -486,14 +483,36 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from)); routine = DatumGetPointer(datum); - if (routine == NULL || !IsA(routine, CopyToRoutine)) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct", - format, handlerOid), - parser_errposition(pstate, defel->location))); - - opts_out->to_routine = routine; + if (is_from) + { + if (routine == NULL || !IsA(routine, CopyFromRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%s(%u) did not return a " + "CopyFromRoutine struct", + format, handlerOid), + parser_errposition( + pstate, defel->location))); + opts_out->from_routine = routine; + } + else + { + if (routine == NULL || !IsA(routine, CopyToRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%s(%u) did not return a " + "CopyToRoutine struct", + format, handlerOid), + parser_errposition( + pstate, defel->location))); + opts_out->to_routine = routine; + } } /* @@ -692,7 +711,11 @@ ProcessCopyOptions(ParseState *pstate, { bool processed = false; - if (!is_from) + if (is_from) + processed = + opts_out->from_routine->CopyFromProcessOption( + cstate, defel); + else processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); if (!processed) ereport(ERROR, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 323e4705d2..ef1bb201c2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -30,6 +30,8 @@ typedef void (*CopyFromEnd_function) (CopyFromState cstate); /* Routines for a COPY FROM format implementation. */ typedef struct CopyFromRoutine { + NodeTag type; + /* * Called for processing one COPY FROM option. This will return false when * the given option is invalid. diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out index 3a24ae7b97..6af69f0eb7 100644 --- a/src/test/modules/test_copy_format/expected/test_copy_format.out +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -1,6 +1,18 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a INT, b INT, c INT); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test FROM stdin WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromProcessOption: "option_before"="before" +NOTICE: CopyFromProcessOption: "option_after"="after" +NOTICE: CopyFromGetFormat +NOTICE: CopyFromStart: natts=3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd COPY public.test TO stdout WITH ( option_before 'before', format 'test_copy_format', diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql index 0eb7ed2e11..94d3c789a0 100644 --- a/src/test/modules/test_copy_format/sql/test_copy_format.sql +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -1,6 +1,12 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a INT, b INT, c INT); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test FROM stdin WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +\. COPY public.test TO stdout WITH ( option_before 'before', format 'test_copy_format', diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index a2219afcde..5e1b40e881 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -18,6 +18,50 @@ PG_MODULE_MAGIC; +static bool +CopyFromProcessOption(CopyFromState cstate, DefElem *defel) +{ + ereport(NOTICE, + (errmsg("CopyFromProcessOption: \"%s\"=\"%s\"", + defel->defname, defGetString(defel)))); + return true; +} + +static int16 +CopyFromGetFormat(CopyFromState cstate) +{ + ereport(NOTICE, (errmsg("CopyFromGetFormat"))); + return 0; +} + +static void +CopyFromStart(CopyFromState cstate, TupleDesc tupDesc) +{ + ereport(NOTICE, (errmsg("CopyFromStart: natts=%d", tupDesc->natts))); +} + +static bool +CopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + ereport(NOTICE, (errmsg("CopyFromOneRow"))); + return false; +} + +static void +CopyFromEnd(CopyFromState cstate) +{ + ereport(NOTICE, (errmsg("CopyFromEnd"))); +} + +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { + .type = T_CopyFromRoutine, + .CopyFromProcessOption = CopyFromProcessOption, + .CopyFromGetFormat = CopyFromGetFormat, + .CopyFromStart = CopyFromStart, + .CopyFromOneRow = CopyFromOneRow, + .CopyFromEnd = CopyFromEnd, +}; + static bool CopyToProcessOption(CopyToState cstate, DefElem *defel) { @@ -71,7 +115,7 @@ test_copy_format(PG_FUNCTION_ARGS) (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false"))); if (is_from) - elog(ERROR, "COPY FROM isn't supported yet"); - - PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); + PG_RETURN_POINTER(&CopyFromRoutineTestCopyFormat); + else + PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); } -- 2.41.0 [application/octet-stream] v8-0010-introduce-contrib-pg_copy_json.patch (16.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/9-v8-0010-introduce-contrib-pg_copy_json.patch) download | inline diff: From 7dc6c1c798178f31728d048d4d528181626b3695 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Sat, 27 Jan 2024 13:34:38 +0800 Subject: [PATCH v8 10/10] introduce contrib/pg_copy_json Signed-off-by: Zhao Junwang <[email protected]> --- contrib/Makefile | 1 + contrib/meson.build | 1 + contrib/pg_copy_json/.gitignore | 4 + contrib/pg_copy_json/Makefile | 23 ++ .../pg_copy_json/expected/pg_copy_json.out | 80 +++++++ contrib/pg_copy_json/meson.build | 34 +++ contrib/pg_copy_json/pg_copy_json--1.0.sql | 9 + contrib/pg_copy_json/pg_copy_json.c | 218 ++++++++++++++++++ contrib/pg_copy_json/pg_copy_json.control | 5 + contrib/pg_copy_json/sql/pg_copy_json.sql | 59 +++++ src/backend/utils/adt/json.c | 5 +- src/include/utils/json.h | 2 + 12 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 contrib/pg_copy_json/.gitignore create mode 100644 contrib/pg_copy_json/Makefile create mode 100644 contrib/pg_copy_json/expected/pg_copy_json.out create mode 100644 contrib/pg_copy_json/meson.build create mode 100644 contrib/pg_copy_json/pg_copy_json--1.0.sql create mode 100644 contrib/pg_copy_json/pg_copy_json.c create mode 100644 contrib/pg_copy_json/pg_copy_json.control create mode 100644 contrib/pg_copy_json/sql/pg_copy_json.sql diff --git a/contrib/Makefile b/contrib/Makefile index da4e2316a3..82cc496aa2 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ pageinspect \ passwordcheck \ pg_buffercache \ + pg_copy_json \ pg_freespacemap \ pg_prewarm \ pg_stat_statements \ diff --git a/contrib/meson.build b/contrib/meson.build index c12dc906ca..38933d15d1 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -45,6 +45,7 @@ subdir('oid2name') subdir('pageinspect') subdir('passwordcheck') subdir('pg_buffercache') +subdir('pg_copy_json') subdir('pgcrypto') subdir('pg_freespacemap') subdir('pg_prewarm') diff --git a/contrib/pg_copy_json/.gitignore b/contrib/pg_copy_json/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/pg_copy_json/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_copy_json/Makefile b/contrib/pg_copy_json/Makefile new file mode 100644 index 0000000000..b0a348d618 --- /dev/null +++ b/contrib/pg_copy_json/Makefile @@ -0,0 +1,23 @@ +# contrib/pg_copy_json//Makefile + +MODULE_big = pg_copy_json +OBJS = \ + $(WIN32RES) \ + pg_copy_json.o +PGFILEDESC = "pg_copy_json - COPY TO JSON (JavaScript Object Notation) format" + +EXTENSION = pg_copy_json +DATA = pg_copy_json--1.0.sql + +REGRESS = test_copy_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_copy_json +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_copy_json/expected/pg_copy_json.out b/contrib/pg_copy_json/expected/pg_copy_json.out new file mode 100644 index 0000000000..73633c2303 --- /dev/null +++ b/contrib/pg_copy_json/expected/pg_copy_json.out @@ -0,0 +1,80 @@ +-- +-- COPY TO JSON +-- +CREATE EXTENSION pg_copy_json; +-- test copying in JSON format with various styles +-- of embedded line ending characters +create temp table copytest ( + style text, + test text, + filler int); +insert into copytest values('DOS',E'abc\r\ndef',1); +insert into copytest values('Unix',E'abc\ndef',2); +insert into copytest values('Mac',E'abc\rdef',3); +insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); +copy copytest to stdout with (format 'json'); +{"style":"DOS","test":"abc\r\ndef","filler":1} +{"style":"Unix","test":"abc\ndef","filler":2} +{"style":"Mac","test":"abc\rdef","filler":3} +{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +-- pg_copy_json do not support COPY FROM +copy copytest from stdout with (format 'json'); +ERROR: cannot use JSON mode in COPY FROM +-- test copying in JSON format with various styles +-- of embedded escaped characters +create temp table copyjsontest ( + id bigserial, + f1 text, + f2 timestamptz); +insert into copyjsontest + select g.i, + CASE WHEN g.i % 2 = 0 THEN + 'line with '' in it: ' || g.i::text + ELSE + 'line with " in it: ' || g.i::text + END, + 'Mon Feb 10 17:32:01 1997 PST' + from generate_series(1,5) as g(i); +insert into copyjsontest (f1) values +(E'aaa\"bbb'::text), +(E'aaa\\bbb'::text), +(E'aaa\/bbb'::text), +(E'aaa\bbbb'::text), +(E'aaa\fbbb'::text), +(E'aaa\nbbb'::text), +(E'aaa\rbbb'::text), +(E'aaa\tbbb'::text); +copy copyjsontest to stdout with (format 'json'); +{"id":1,"f1":"line with \" in it: 1","f2":"1997-02-10T17:32:01-08:00"} +{"id":2,"f1":"line with ' in it: 2","f2":"1997-02-10T17:32:01-08:00"} +{"id":3,"f1":"line with \" in it: 3","f2":"1997-02-10T17:32:01-08:00"} +{"id":4,"f1":"line with ' in it: 4","f2":"1997-02-10T17:32:01-08:00"} +{"id":5,"f1":"line with \" in it: 5","f2":"1997-02-10T17:32:01-08:00"} +{"id":1,"f1":"aaa\"bbb","f2":null} +{"id":2,"f1":"aaa\\bbb","f2":null} +{"id":3,"f1":"aaa/bbb","f2":null} +{"id":4,"f1":"aaa\bbbb","f2":null} +{"id":5,"f1":"aaa\fbbb","f2":null} +{"id":6,"f1":"aaa\nbbb","f2":null} +{"id":7,"f1":"aaa\rbbb","f2":null} +{"id":8,"f1":"aaa\tbbb","f2":null} +-- test force array +copy copytest to stdout (format 'json', force_array); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format 'json', force_array true); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format 'json', force_array false); +{"style":"DOS","test":"abc\r\ndef","filler":1} +{"style":"Unix","test":"abc\ndef","filler":2} +{"style":"Mac","test":"abc\rdef","filler":3} +{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} diff --git a/contrib/pg_copy_json/meson.build b/contrib/pg_copy_json/meson.build new file mode 100644 index 0000000000..71f9338267 --- /dev/null +++ b/contrib/pg_copy_json/meson.build @@ -0,0 +1,34 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +pg_copy_json_sources = files( + 'pg_copy_json.c', +) + +if host_system == 'windows' + pg_copy_json_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'pg_copy_json', + '--FILEDESC', 'pg_copy_json - COPY TO JSON format',]) +endif + +pg_copy_json = shared_module('pg_copy_json', + pg_copy_json_sources, + kwargs: contrib_mod_args, +) +contrib_targets += pg_copy_json + +install_data( + 'pg_copy_json--1.0.sql', + 'pg_copy_json.control', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'pg_copy_json', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'pg_copy_json', + ], + }, +} diff --git a/contrib/pg_copy_json/pg_copy_json--1.0.sql b/contrib/pg_copy_json/pg_copy_json--1.0.sql new file mode 100644 index 0000000000..d738a1e7e9 --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json--1.0.sql @@ -0,0 +1,9 @@ +/* contrib/pg_copy_json/copy_json--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_copy_json" to load this file. \quit + +CREATE FUNCTION pg_catalog.json(internal) + RETURNS copy_handler + AS 'MODULE_PATHNAME', 'copy_json' + LANGUAGE C; diff --git a/contrib/pg_copy_json/pg_copy_json.c b/contrib/pg_copy_json/pg_copy_json.c new file mode 100644 index 0000000000..cbfdee8e8b --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json.c @@ -0,0 +1,218 @@ +/*-------------------------------------------------------------------------- + * + * pg_copy_json.c + * COPY TO JSON (JavaScript Object Notation) format. + * + * Portions Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/test_copy_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/defrem.h" +#include "funcapi.h" +#include "libpq/libpq.h" +#include "libpq/pqformat.h" +#include "utils/json.h" + +PG_MODULE_MAGIC; + +typedef struct +{ + /* + * Force output of square brackets as array decorations at the beginning + * and end of output, with commas between the rows. + */ + bool force_array; + bool force_array_specified; + + /* need delimiter to start next json array element */ + bool json_row_delim_needed; +} CopyJsonData; + +static inline void +InitCopyJsonData(CopyJsonData *p) +{ + Assert(p); + p->force_array = false; + p->force_array_specified = false; + p->json_row_delim_needed = false; +} + +static void +CopyToJsonSendEndOfRow(CopyToState cstate) +{ + switch (cstate->copy_dest) + { + case COPY_DEST_FILE: + /* Default line termination depends on platform */ +#ifndef WIN32 + CopySendChar(cstate, '\n'); +#else + CopySendString(cstate, "\r\n"); +#endif + break; + case COPY_DEST_FRONTEND: + /* The FE/BE protocol uses \n as newline for all platforms */ + CopySendChar(cstate, '\n'); + break; + default: + break; + } + CopyToStateFlush(cstate); +} + +static bool +CopyToJsonProcessOption(CopyToState cstate, DefElem *defel) +{ + CopyJsonData *p; + + if (cstate->opaque == NULL) + { + MemoryContext oldcontext; + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + cstate->opaque = palloc0(sizeof(CopyJsonData)); + MemoryContextSwitchTo(oldcontext); + InitCopyJsonData(cstate->opaque); + } + + p = (CopyJsonData *)cstate->opaque; + + if (strcmp(defel->defname, "force_array") == 0) + { + if (p->force_array_specified) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("CopyToJsonProcessOption: redundant options \"%s\"=\"%s\"", + defel->defname, defGetString(defel))); + p->force_array_specified = true; + p->force_array = defGetBoolean(defel); + + return true; + } + + return false; +} + +static void +CopyToJsonSendCopyBegin(CopyToState cstate) +{ + StringInfoData buf; + int16 format = 0; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + /* + * JSON mode is always one non-binary column + */ + pq_sendint16(&buf, 1); + pq_sendint16(&buf, 0); + pq_endmessage(&buf); +} + +static void +CopyToJsonStart(CopyToState cstate, TupleDesc tupDesc) +{ + CopyJsonData *p; + + if (cstate->opaque == NULL) + { + MemoryContext oldcontext; + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + cstate->opaque = palloc0(sizeof(CopyJsonData)); + MemoryContextSwitchTo(oldcontext); + InitCopyJsonData(cstate->opaque); + } + + /* No need to alloc cstate->out_functions */ + + p = (CopyJsonData *)cstate->opaque; + + /* If FORCE_ARRAY has been specified send the open bracket. */ + if (p->force_array) + { + CopySendChar(cstate, '['); + CopyToJsonSendEndOfRow(cstate); + } +} + +static void +CopyToJsonOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + Datum rowdata; + StringInfo result; + CopyJsonData *p; + + Assert(cstate->opaque); + p = (CopyJsonData *)cstate->opaque; + + if(!cstate->rel) + { + for (int i = 0; i < slot->tts_tupleDescriptor->natts; i++) + { + /* Flat-copy the attribute array */ + memcpy(TupleDescAttr(slot->tts_tupleDescriptor, i), + TupleDescAttr(cstate->queryDesc->tupDesc, i), + 1 * sizeof(FormData_pg_attribute)); + } + BlessTupleDesc(slot->tts_tupleDescriptor); + } + rowdata = ExecFetchSlotHeapTupleDatum(slot); + result = makeStringInfo(); + composite_to_json(rowdata, result, false); + + if (p->json_row_delim_needed) + CopySendChar(cstate, ','); + else if (p->force_array) + { + /* first row needs no delimiter */ + CopySendChar(cstate, ' '); + p->json_row_delim_needed = true; + } + CopySendData(cstate, result->data, result->len); + CopyToJsonSendEndOfRow(cstate); +} + +static void +CopyToJsonEnd(CopyToState cstate) +{ + CopyJsonData *p; + + Assert(cstate->opaque); + p = (CopyJsonData *)cstate->opaque; + + /* If FORCE_ARRAY has been specified send the close bracket. */ + if (p->force_array) + { + CopySendChar(cstate, ']'); + CopyToJsonSendEndOfRow(cstate); + } +} + +static const CopyToRoutine CopyToRoutineJson = { + .type = T_CopyToRoutine, + .CopyToProcessOption = CopyToJsonProcessOption, + .CopyToSendCopyBegin = CopyToJsonSendCopyBegin, + .CopyToStart = CopyToJsonStart, + .CopyToOneRow = CopyToJsonOneRow, + .CopyToEnd = CopyToJsonEnd, +}; + +PG_FUNCTION_INFO_V1(copy_json); +Datum +copy_json(PG_FUNCTION_ARGS) +{ + bool is_from = PG_GETARG_BOOL(0); + + if (is_from) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use JSON mode in COPY FROM"))); + + PG_RETURN_POINTER(&CopyToRoutineJson); +} diff --git a/contrib/pg_copy_json/pg_copy_json.control b/contrib/pg_copy_json/pg_copy_json.control new file mode 100644 index 0000000000..90b0a74603 --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json.control @@ -0,0 +1,5 @@ +# pg_copy_json extension +comment = 'COPY TO JSON format' +default_version = '1.0' +module_pathname = '$libdir/pg_copy_json' +relocatable = true diff --git a/contrib/pg_copy_json/sql/pg_copy_json.sql b/contrib/pg_copy_json/sql/pg_copy_json.sql new file mode 100644 index 0000000000..73e7e514ac --- /dev/null +++ b/contrib/pg_copy_json/sql/pg_copy_json.sql @@ -0,0 +1,59 @@ +-- +-- COPY TO JSON +-- + +CREATE EXTENSION pg_copy_json; + +-- test copying in JSON format with various styles +-- of embedded line ending characters + +create temp table copytest ( + style text, + test text, + filler int); + +insert into copytest values('DOS',E'abc\r\ndef',1); +insert into copytest values('Unix',E'abc\ndef',2); +insert into copytest values('Mac',E'abc\rdef',3); +insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); + +copy copytest to stdout with (format 'json'); + +-- pg_copy_json do not support COPY FROM +copy copytest from stdout with (format 'json'); + +-- test copying in JSON format with various styles +-- of embedded escaped characters + +create temp table copyjsontest ( + id bigserial, + f1 text, + f2 timestamptz); + +insert into copyjsontest + select g.i, + CASE WHEN g.i % 2 = 0 THEN + 'line with '' in it: ' || g.i::text + ELSE + 'line with " in it: ' || g.i::text + END, + 'Mon Feb 10 17:32:01 1997 PST' + from generate_series(1,5) as g(i); + +insert into copyjsontest (f1) values +(E'aaa\"bbb'::text), +(E'aaa\\bbb'::text), +(E'aaa\/bbb'::text), +(E'aaa\bbbb'::text), +(E'aaa\fbbb'::text), +(E'aaa\nbbb'::text), +(E'aaa\rbbb'::text), +(E'aaa\tbbb'::text); + +copy copyjsontest to stdout with (format 'json'); + +-- test force array + +copy copytest to stdout (format 'json', force_array); +copy copytest to stdout (format 'json', force_array true); +copy copytest to stdout (format 'json', force_array false); diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c index d719a61f16..fabd4e611e 100644 --- a/src/backend/utils/adt/json.c +++ b/src/backend/utils/adt/json.c @@ -83,8 +83,6 @@ typedef struct JsonAggState JsonUniqueBuilderState unique_check; } JsonAggState; -static void composite_to_json(Datum composite, StringInfo result, - bool use_line_feeds); static void array_dim_to_json(StringInfo result, int dim, int ndims, int *dims, Datum *vals, bool *nulls, int *valcount, JsonTypeCategory tcategory, Oid outfuncoid, @@ -507,8 +505,9 @@ array_to_json_internal(Datum array, StringInfo result, bool use_line_feeds) /* * Turn a composite / record into JSON. + * Exported so COPY TO can use it. */ -static void +void composite_to_json(Datum composite, StringInfo result, bool use_line_feeds) { HeapTupleHeader td; diff --git a/src/include/utils/json.h b/src/include/utils/json.h index 6d7f1b387d..d5631171ad 100644 --- a/src/include/utils/json.h +++ b/src/include/utils/json.h @@ -17,6 +17,8 @@ #include "lib/stringinfo.h" /* functions in json.c */ +extern void composite_to_json(Datum composite, StringInfo result, + bool use_line_feeds); extern void escape_json(StringInfo buf, const char *str); extern char *JsonEncodeDateTime(char *buf, Datum value, Oid typid, const int *tzp); -- 2.41.0 [application/octet-stream] v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch (4.5K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/10-v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch) download | inline diff: From 3e847de1acb2fd6966ef01192204448711ca3d5e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 14:19:08 +0900 Subject: [PATCH v8 08/10] Add support for implementing custom COPY FROM format as extension * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index a78a790060..f8a194635d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b7e8f627bf..22accc83ab 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -314,8 +314,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + /* * Represents the different dest cases we need to worry about at * the bottom level -- 2.41.0 [application/octet-stream] v8-0007-Export-CopyFromStateData.patch (17.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/11-v8-0007-Export-CopyFromStateData.patch) download | inline diff: From 1ed575fda7f196ea411e9e53dd9c0739f160fb78 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 14:16:29 +0900 Subject: [PATCH v8 07/10] Export CopyFromStateData It's for custom COPY FROM format handlers implemented as extension. This just moves codes. This doesn't change codes except CopySource enum values. CopySource enum values changes aren't required but I did like I did for CopyDest enum values. I changed COPY_ prefix to COPY_SOURCE_ prefix. For example, COPY_FILE to COPY_SOURCE_FILE. Note that this change isn't enough to implement a custom COPY FROM format handler as extension. We'll do the followings in a subsequent commit: 1. Add an opaque space for custom COPY FROM format handler 2. Export CopyReadBinaryData() to read the next data --- src/backend/commands/copyfrom.c | 4 +- src/backend/commands/copyfromparse.c | 10 +- src/include/commands/copy.h | 2 - src/include/commands/copyapi.h | 156 ++++++++++++++++++++++- src/include/commands/copyfrom_internal.h | 150 ---------------------- 5 files changed, 162 insertions(+), 160 deletions(-) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index d556ebb5d6..b4ac7cbd2c 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1710,7 +1710,7 @@ BeginCopyFrom(ParseState *pstate, pg_encoding_to_char(GetDatabaseEncoding())))); } - cstate->copy_src = COPY_FILE; /* default */ + cstate->copy_src = COPY_SOURCE_FILE; /* default */ cstate->whereClause = whereClause; @@ -1829,7 +1829,7 @@ BeginCopyFrom(ParseState *pstate, if (data_source_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_src = COPY_CALLBACK; + cstate->copy_src = COPY_SOURCE_CALLBACK; cstate->data_source_cb = data_source_cb; } else if (pipe) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 49632f75e4..a78a790060 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -181,7 +181,7 @@ ReceiveCopyBegin(CopyFromState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_src = COPY_FRONTEND; + cstate->copy_src = COPY_SOURCE_FRONTEND; cstate->fe_msgbuf = makeStringInfo(); /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); @@ -249,7 +249,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) switch (cstate->copy_src) { - case COPY_FILE: + case COPY_SOURCE_FILE: bytesread = fread(databuf, 1, maxread, cstate->copy_file); if (ferror(cstate->copy_file)) ereport(ERROR, @@ -258,7 +258,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) if (bytesread == 0) cstate->raw_reached_eof = true; break; - case COPY_FRONTEND: + case COPY_SOURCE_FRONTEND: while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) { int avail; @@ -341,7 +341,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) bytesread += avail; } break; - case COPY_CALLBACK: + case COPY_SOURCE_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); break; } @@ -1099,7 +1099,7 @@ CopyReadLine(CopyFromState cstate) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (cstate->copy_src == COPY_SOURCE_FRONTEND) { int inbytes; diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index df29d42555..cd41d32074 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,8 +20,6 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); - extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index ef1bb201c2..b7e8f627bf 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,11 +14,12 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "commands/trigger.h" #include "executor/execdesc.h" #include "executor/tuptable.h" +#include "nodes/miscnodes.h" #include "nodes/parsenodes.h" -/* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); @@ -162,6 +163,159 @@ typedef struct CopyFormatOptions CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; + +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_SOURCE_FILE, /* from file (or a piped program) */ + COPY_SOURCE_FRONTEND, /* from frontend */ + COPY_SOURCE_CALLBACK, /* from callback function */ +} CopySource; + +/* + * Represents the end-of-line terminator type of the input + */ +typedef enum EolType +{ + EOL_UNKNOWN, + EOL_NL, + EOL_CR, + EOL_CRNL, +} EolType; + +typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); + +/* + * This struct contains all the state variables used throughout a COPY FROM + * operation. + */ +typedef struct CopyFromStateData +{ + /* low-level state data */ + CopySource copy_src; /* type of copy source */ + FILE *copy_file; /* used if copy_src == COPY_FILE */ + StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ + + EolType eol_type; /* EOL type of input */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + Oid conversion_proc; /* encoding conversion function */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + CopyFormatOptions opts; + bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ + Node *whereClause; /* WHERE condition (or NULL) */ + + /* these are just for error messages, see CopyFromErrorCallback */ + const char *cur_relname; /* table name for error messages */ + uint64 cur_lineno; /* line number for error messages */ + const char *cur_attname; /* current att for error messages */ + const char *cur_attval; /* current att value for error messages */ + bool relname_only; /* don't output line number, att, etc. */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + AttrNumber num_defaults; /* count of att that are missing and have + * default value */ + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapper during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ + int *defmap; /* array of default att numbers related to + * missing att */ + ExprState **defexprs; /* array of default att expressions for all + * att */ + bool *defaults; /* if DEFAULT marker was found for + * corresponding att */ + bool volatile_defexprs; /* is any of defexprs volatile? */ + List *range_table; /* single element list of RangeTblEntry */ + List *rteperminfos; /* single element list of RTEPermissionInfo */ + ExprState *qualexpr; + + TransitionCaptureState *transition_capture; + + /* + * These variables are used to reduce overhead in COPY FROM. + * + * attribute_buf holds the separated, de-escaped text for each field of + * the current line. The CopyReadAttributes functions return arrays of + * pointers into this buffer. We avoid palloc/pfree overhead by re-using + * the buffer on each cycle. + * + * In binary COPY FROM, attribute_buf holds the binary data for the + * current field, but the usage is otherwise similar. + */ + StringInfoData attribute_buf; + + /* field raw data pointers found by COPY FROM */ + + int max_fields; + char **raw_fields; + + /* + * Similarly, line_buf holds the whole input line being processed. The + * input cycle is first to read the whole line into line_buf, and then + * extract the individual attribute fields into attribute_buf. line_buf + * is preserved unmodified so that we can display it in error messages if + * appropriate. (In binary mode, line_buf is not used.) + */ + StringInfoData line_buf; + bool line_buf_valid; /* contains the row being processed? */ + + /* + * input_buf holds input data, already converted to database encoding. + * + * In text mode, CopyReadLine parses this data sufficiently to locate line + * boundaries, then transfers the data to line_buf. We guarantee that + * there is a \0 at input_buf[input_buf_len] at all times. (In binary + * mode, input_buf is not used.) + * + * If encoding conversion is not required, input_buf is not a separate + * buffer but points directly to raw_buf. In that case, input_buf_len + * tracks the number of bytes that have been verified as valid in the + * database encoding, and raw_buf_len is the total number of bytes stored + * in the buffer. + */ +#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ + char *input_buf; + int input_buf_index; /* next byte to process */ + int input_buf_len; /* total # of bytes stored */ + bool input_reached_eof; /* true if we reached EOF */ + bool input_reached_error; /* true if a conversion error happened */ + /* Shorthand for number of unconsumed bytes available in input_buf */ +#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) + + /* + * raw_buf holds raw input data read from the data source (file or client + * connection), not yet converted to the database encoding. Like with + * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. + */ +#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ + char *raw_buf; + int raw_buf_index; /* next byte to process */ + int raw_buf_len; /* total # of bytes stored */ + bool raw_reached_eof; /* true if we reached EOF */ + + /* Shorthand for number of unconsumed bytes available in raw_buf */ +#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) + + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyFromStateData; + /* * Represents the different dest cases we need to worry about at * the bottom level diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 921c1513f7..f8f6120255 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -18,28 +18,6 @@ #include "commands/trigger.h" #include "nodes/miscnodes.h" -/* - * Represents the different source cases we need to worry about at - * the bottom level - */ -typedef enum CopySource -{ - COPY_FILE, /* from file (or a piped program) */ - COPY_FRONTEND, /* from frontend */ - COPY_CALLBACK, /* from callback function */ -} CopySource; - -/* - * Represents the end-of-line terminator type of the input - */ -typedef enum EolType -{ - EOL_UNKNOWN, - EOL_NL, - EOL_CR, - EOL_CRNL, -} EolType; - /* * Represents the insert method to be used during COPY FROM. */ @@ -52,134 +30,6 @@ typedef enum CopyInsertMethod * ExecForeignBatchInsert only if valid */ } CopyInsertMethod; -/* - * This struct contains all the state variables used throughout a COPY FROM - * operation. - */ -typedef struct CopyFromStateData -{ - /* low-level state data */ - CopySource copy_src; /* type of copy source */ - FILE *copy_file; /* used if copy_src == COPY_FILE */ - StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ - - EolType eol_type; /* EOL type of input */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - Oid conversion_proc; /* encoding conversion function */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; - bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ - Node *whereClause; /* WHERE condition (or NULL) */ - - /* these are just for error messages, see CopyFromErrorCallback */ - const char *cur_relname; /* table name for error messages */ - uint64 cur_lineno; /* line number for error messages */ - const char *cur_attname; /* current att for error messages */ - const char *cur_attval; /* current att value for error messages */ - bool relname_only; /* don't output line number, att, etc. */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - AttrNumber num_defaults; /* count of att that are missing and have - * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ - ErrorSaveContext *escontext; /* soft error trapper during in_functions - * execution */ - uint64 num_errors; /* total number of rows which contained soft - * errors */ - int *defmap; /* array of default att numbers related to - * missing att */ - ExprState **defexprs; /* array of default att expressions for all - * att */ - bool *defaults; /* if DEFAULT marker was found for - * corresponding att */ - bool volatile_defexprs; /* is any of defexprs volatile? */ - List *range_table; /* single element list of RangeTblEntry */ - List *rteperminfos; /* single element list of RTEPermissionInfo */ - ExprState *qualexpr; - - TransitionCaptureState *transition_capture; - - /* - * These variables are used to reduce overhead in COPY FROM. - * - * attribute_buf holds the separated, de-escaped text for each field of - * the current line. The CopyReadAttributes functions return arrays of - * pointers into this buffer. We avoid palloc/pfree overhead by re-using - * the buffer on each cycle. - * - * In binary COPY FROM, attribute_buf holds the binary data for the - * current field, but the usage is otherwise similar. - */ - StringInfoData attribute_buf; - - /* field raw data pointers found by COPY FROM */ - - int max_fields; - char **raw_fields; - - /* - * Similarly, line_buf holds the whole input line being processed. The - * input cycle is first to read the whole line into line_buf, and then - * extract the individual attribute fields into attribute_buf. line_buf - * is preserved unmodified so that we can display it in error messages if - * appropriate. (In binary mode, line_buf is not used.) - */ - StringInfoData line_buf; - bool line_buf_valid; /* contains the row being processed? */ - - /* - * input_buf holds input data, already converted to database encoding. - * - * In text mode, CopyReadLine parses this data sufficiently to locate line - * boundaries, then transfers the data to line_buf. We guarantee that - * there is a \0 at input_buf[input_buf_len] at all times. (In binary - * mode, input_buf is not used.) - * - * If encoding conversion is not required, input_buf is not a separate - * buffer but points directly to raw_buf. In that case, input_buf_len - * tracks the number of bytes that have been verified as valid in the - * database encoding, and raw_buf_len is the total number of bytes stored - * in the buffer. - */ -#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ - char *input_buf; - int input_buf_index; /* next byte to process */ - int input_buf_len; /* total # of bytes stored */ - bool input_reached_eof; /* true if we reached EOF */ - bool input_reached_error; /* true if a conversion error happened */ - /* Shorthand for number of unconsumed bytes available in input_buf */ -#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) - - /* - * raw_buf holds raw input data read from the data source (file or client - * connection), not yet converted to the database encoding. Like with - * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. - */ -#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ - char *raw_buf; - int raw_buf_index; /* next byte to process */ - int raw_buf_len; /* total # of bytes stored */ - bool raw_reached_eof; /* true if we reached EOF */ - - /* Shorthand for number of unconsumed bytes available in raw_buf */ -#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) - - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; - extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); -- 2.41.0 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 02:41 Masahiko Sawada <[email protected]> parent: Junwang Zhao <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2024-01-29 02:41 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > Junwang Zhao <[email protected]> wrote: > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > single option, and store the options in the opaque field, but it can not > > > check the relation of two options, for example, considering json format, > > > the `header` option can not be handled by these two functions. > > > > > > I want to find a way when the user specifies the header option, customer > > > handler can error out. > > > > Ah, you want to use a built-in option (such as "header") > > value from a custom handler, right? Hmm, it may be better > > that we call CopyToProcessOption()/CopyFromProcessOption() > > for all options including built-in options. > > > Hmm, still I don't think it can handle all cases, since we don't know > the sequence of the options, we need all the options been parsed > before we check the compatibility of the options, or customer > handlers will need complicated logic to resolve that, which might > lead to ugly code :( > Does it make sense to pass only non-builtin options to the custom format callback after parsing and evaluating the builtin options? That is, we parse and evaluate only the builtin options and populate opts_out first, then pass each rest option to the custom format handler callback. The callback can refer to the builtin option values. The callback is expected to return false if the passed option is not supported. If one of the builtin formats is specified and the rest options list has at least one option, we raise "option %s not recognized" error. IOW it's the core's responsibility to ranse the "option %s not recognized" error, which is in order to raise a consistent error message. Also, I think the core should check the redundant options including bultiin and custom options. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:10 Junwang Zhao <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Junwang Zhao @ 2024-01-29 03:10 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > Hi, > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > Junwang Zhao <[email protected]> wrote: > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > single option, and store the options in the opaque field, but it can not > > > > check the relation of two options, for example, considering json format, > > > > the `header` option can not be handled by these two functions. > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > handler can error out. > > > > > > Ah, you want to use a built-in option (such as "header") > > > value from a custom handler, right? Hmm, it may be better > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > for all options including built-in options. > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > the sequence of the options, we need all the options been parsed > > before we check the compatibility of the options, or customer > > handlers will need complicated logic to resolve that, which might > > lead to ugly code :( > > > > Does it make sense to pass only non-builtin options to the custom > format callback after parsing and evaluating the builtin options? That > is, we parse and evaluate only the builtin options and populate > opts_out first, then pass each rest option to the custom format > handler callback. The callback can refer to the builtin option values. Yeah, I think this makes sense. > The callback is expected to return false if the passed option is not > supported. If one of the builtin formats is specified and the rest > options list has at least one option, we raise "option %s not > recognized" error. IOW it's the core's responsibility to ranse the > "option %s not recognized" error, which is in order to raise a > consistent error message. Also, I think the core should check the > redundant options including bultiin and custom options. It would be good that core could check all the redundant options, but where should core do the book-keeping of all the options? I have no idea about this, in my implementation of pg_copy_json extension, I handle redundant options by adding a xxx_specified field for each xxx. > > Regards, > > -- > Masahiko Sawada > Amazon Web Services: https://aws.amazon.com -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:21 Masahiko Sawada <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2024-01-29 03:21 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote: > > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > > > Hi, > > > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > > Junwang Zhao <[email protected]> wrote: > > > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > > single option, and store the options in the opaque field, but it can not > > > > > check the relation of two options, for example, considering json format, > > > > > the `header` option can not be handled by these two functions. > > > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > > handler can error out. > > > > > > > > Ah, you want to use a built-in option (such as "header") > > > > value from a custom handler, right? Hmm, it may be better > > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > > for all options including built-in options. > > > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > > the sequence of the options, we need all the options been parsed > > > before we check the compatibility of the options, or customer > > > handlers will need complicated logic to resolve that, which might > > > lead to ugly code :( > > > > > > > Does it make sense to pass only non-builtin options to the custom > > format callback after parsing and evaluating the builtin options? That > > is, we parse and evaluate only the builtin options and populate > > opts_out first, then pass each rest option to the custom format > > handler callback. The callback can refer to the builtin option values. > > Yeah, I think this makes sense. > > > The callback is expected to return false if the passed option is not > > supported. If one of the builtin formats is specified and the rest > > options list has at least one option, we raise "option %s not > > recognized" error. IOW it's the core's responsibility to ranse the > > "option %s not recognized" error, which is in order to raise a > > consistent error message. Also, I think the core should check the > > redundant options including bultiin and custom options. > > It would be good that core could check all the redundant options, > but where should core do the book-keeping of all the options? I have > no idea about this, in my implementation of pg_copy_json extension, > I handle redundant options by adding a xxx_specified field for each > xxx. What I imagined is that while parsing the all specified options, we evaluate builtin options and we add non-builtin options to another list. Then when parsing a non-builtin option, we check if this option already exists in the list. If there is, we raise the "option %s not recognized" error.". Once we complete checking all options, we pass each option in the list to the callback. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:37 Junwang Zhao <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Junwang Zhao @ 2024-01-29 03:37 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 11:22 AM Masahiko Sawada <[email protected]> wrote: > > On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote: > > > > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > > > > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > > > > > Hi, > > > > > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > > > Junwang Zhao <[email protected]> wrote: > > > > > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > > > single option, and store the options in the opaque field, but it can not > > > > > > check the relation of two options, for example, considering json format, > > > > > > the `header` option can not be handled by these two functions. > > > > > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > > > handler can error out. > > > > > > > > > > Ah, you want to use a built-in option (such as "header") > > > > > value from a custom handler, right? Hmm, it may be better > > > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > > > for all options including built-in options. > > > > > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > > > the sequence of the options, we need all the options been parsed > > > > before we check the compatibility of the options, or customer > > > > handlers will need complicated logic to resolve that, which might > > > > lead to ugly code :( > > > > > > > > > > Does it make sense to pass only non-builtin options to the custom > > > format callback after parsing and evaluating the builtin options? That > > > is, we parse and evaluate only the builtin options and populate > > > opts_out first, then pass each rest option to the custom format > > > handler callback. The callback can refer to the builtin option values. > > > > Yeah, I think this makes sense. > > > > > The callback is expected to return false if the passed option is not > > > supported. If one of the builtin formats is specified and the rest > > > options list has at least one option, we raise "option %s not > > > recognized" error. IOW it's the core's responsibility to ranse the > > > "option %s not recognized" error, which is in order to raise a > > > consistent error message. Also, I think the core should check the > > > redundant options including bultiin and custom options. > > > > It would be good that core could check all the redundant options, > > but where should core do the book-keeping of all the options? I have > > no idea about this, in my implementation of pg_copy_json extension, > > I handle redundant options by adding a xxx_specified field for each > > xxx. > > What I imagined is that while parsing the all specified options, we > evaluate builtin options and we add non-builtin options to another > list. Then when parsing a non-builtin option, we check if this option > already exists in the list. If there is, we raise the "option %s not > recognized" error.". Once we complete checking all options, we pass > each option in the list to the callback. LGTM. > > Regards, > > -- > Masahiko Sawada > Amazon Web Services: https://aws.amazon.com -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 06:03 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-29 06:03 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800, Junwang Zhao <[email protected]> wrote: > I have been working on a *COPY TO JSON* extension since yesterday, > which is based on your V6 patch set, I'd like to give you more input > so you can make better decisions about the implementation(with only > pg-copy-arrow you might not get everything considered). Thanks! > 0009 is some changes made by me, I changed CopyToGetFormat to > CopyToSendCopyBegin because pg_copy_json need to send different bytes > in SendCopyBegin, get the format code along is not enough Oh, I haven't cared about the case. How about the following API instead? static void SendCopyBegin(CopyToState cstate) { StringInfoData buf; pq_beginmessage(&buf, PqMsg_CopyOutResponse); cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf); pq_endmessage(&buf); cstate->copy_dest = COPY_FRONTEND; } static void CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf) { int16 format = 0; pq_sendbyte(&buf, format); /* overall format */ /* * JSON mode is always one non-binary column */ pq_sendint16(&buf, 1); pq_sendint16(&buf, format); } > 00010 is the pg_copy_json extension, I think this should be a good > case which can utilize the *extendable copy format* feature It seems that it's convenient that we have one more callback for initializing CopyToState::opaque. It's called only once when Copy{To,From}Routine is chosen: typedef struct CopyToRoutine { void (*CopyToInit) (CopyToState cstate); ... }; void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options) { ... foreach(option, options) { DefElem *defel = lfirst_node(DefElem, option); if (strcmp(defel->defname, "format") == 0) { ... opts_out->to_routine = &CopyToRoutineXXX; opts_out->to_routine->CopyToInit(cstate); ... } } ... } > maybe we > should delete copy_test_format if we have this extension as an > example? I haven't read the COPY TO format json thread[1] carefully (sorry), but we may add the JSON format as a built-in format. If we do it, copy_test_format is useful to test the extension API. [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m... Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 06:48 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Junwang Zhao @ 2024-01-29 06:48 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 2:03 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800, > Junwang Zhao <[email protected]> wrote: > > > I have been working on a *COPY TO JSON* extension since yesterday, > > which is based on your V6 patch set, I'd like to give you more input > > so you can make better decisions about the implementation(with only > > pg-copy-arrow you might not get everything considered). > > Thanks! > > > 0009 is some changes made by me, I changed CopyToGetFormat to > > CopyToSendCopyBegin because pg_copy_json need to send different bytes > > in SendCopyBegin, get the format code along is not enough > > Oh, I haven't cared about the case. > How about the following API instead? > > static void > SendCopyBegin(CopyToState cstate) > { > StringInfoData buf; > > pq_beginmessage(&buf, PqMsg_CopyOutResponse); > cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf); > pq_endmessage(&buf); > cstate->copy_dest = COPY_FRONTEND; > } > > static void > CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf) > { > int16 format = 0; > > pq_sendbyte(&buf, format); /* overall format */ > /* > * JSON mode is always one non-binary column > */ > pq_sendint16(&buf, 1); > pq_sendint16(&buf, format); > } Make sense to me. > > > 00010 is the pg_copy_json extension, I think this should be a good > > case which can utilize the *extendable copy format* feature > > It seems that it's convenient that we have one more callback > for initializing CopyToState::opaque. It's called only once > when Copy{To,From}Routine is chosen: > > typedef struct CopyToRoutine > { > void (*CopyToInit) (CopyToState cstate); > ... > }; I like this, we can alloc private data in this hook. > > void > ProcessCopyOptions(ParseState *pstate, > CopyFormatOptions *opts_out, > bool is_from, > void *cstate, > List *options) > { > ... > foreach(option, options) > { > DefElem *defel = lfirst_node(DefElem, option); > > if (strcmp(defel->defname, "format") == 0) > { > ... > opts_out->to_routine = &CopyToRoutineXXX; > opts_out->to_routine->CopyToInit(cstate); > ... > } > } > ... > } > > > > maybe we > > should delete copy_test_format if we have this extension as an > > example? > > I haven't read the COPY TO format json thread[1] carefully > (sorry), but we may add the JSON format as a built-in > format. If we do it, copy_test_format is useful to test the > extension API. Yeah, maybe, I have no strong opinion here, pg_copy_json is just a toy extension for discussion. > > [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m... > > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 09:45 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-29 09:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800, Junwang Zhao <[email protected]> wrote: >> > > Does it make sense to pass only non-builtin options to the custom >> > > format callback after parsing and evaluating the builtin options? That >> > > is, we parse and evaluate only the builtin options and populate >> > > opts_out first, then pass each rest option to the custom format >> > > handler callback. The callback can refer to the builtin option values. >> >> What I imagined is that while parsing the all specified options, we >> evaluate builtin options and we add non-builtin options to another >> list. Then when parsing a non-builtin option, we check if this option >> already exists in the list. If there is, we raise the "option %s not >> recognized" error.". Once we complete checking all options, we pass >> each option in the list to the callback. I implemented this idea and the following ideas: 1. Add init callback for initialization 2. Change GetFormat() to FillCopyXXXResponse() because JSON format always use 1 column 3. FROM only: Eliminate more cstate->opts.csv_mode branches (This is for performance.) See the attached v9 patch set for details. Changes since v7: 0001: * Move CopyToProcessOption() calls to the end of ProcessCopyOptions() for easy to option validation * Add CopyToState::CopyToInit() and call it in ProcessCopyOptionFormatTo() * Change CopyToState::CopyToGetFormat() to CopyToState::CopyToFillCopyOutResponse() and use it in SendCopyBegin() 0002: * Move CopyFromProcessOption() calls to the end of ProcessCopyOptions() for easy to option validation * Add CopyFromState::CopyFromInit() and call it in ProcessCopyOptionFormatFrom() * Change CopyFromState::CopyFromGetFormat() to CopyFromState::CopyFromFillCopyOutResponse() and use it in ReceiveCopyBegin() * Rename NextCopyFromRawFields() to NextCopyFromRawFieldsInternal() and pass the read attributes callback explicitly to eliminate more cstate->opts.csv_mode branches Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 02:11 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2024-01-30 02:11 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 6:45 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800, > Junwang Zhao <[email protected]> wrote: > > >> > > Does it make sense to pass only non-builtin options to the custom > >> > > format callback after parsing and evaluating the builtin options? That > >> > > is, we parse and evaluate only the builtin options and populate > >> > > opts_out first, then pass each rest option to the custom format > >> > > handler callback. The callback can refer to the builtin option values. > >> > >> What I imagined is that while parsing the all specified options, we > >> evaluate builtin options and we add non-builtin options to another > >> list. Then when parsing a non-builtin option, we check if this option > >> already exists in the list. If there is, we raise the "option %s not > >> recognized" error.". Once we complete checking all options, we pass > >> each option in the list to the callback. > > I implemented this idea and the following ideas: > > 1. Add init callback for initialization > 2. Change GetFormat() to FillCopyXXXResponse() > because JSON format always use 1 column > 3. FROM only: Eliminate more cstate->opts.csv_mode branches > (This is for performance.) > > See the attached v9 patch set for details. Changes since v7: > > 0001: > > * Move CopyToProcessOption() calls to the end of > ProcessCopyOptions() for easy to option validation > * Add CopyToState::CopyToInit() and call it in > ProcessCopyOptionFormatTo() > * Change CopyToState::CopyToGetFormat() to > CopyToState::CopyToFillCopyOutResponse() and use it in > SendCopyBegin() Thank you for updating the patch! Here are comments on 0001 patch: --- + if (!format_specified) + /* Set the default format. */ + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); + I think we can pass "text" in this case instead of NULL. That way, ProcessCopyOptionFormatTo doesn't need to handle NULL case. We need curly brackets for this "if branch" as follows: if (!format_specifed) { /* Set the default format. */ ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); } --- + /* Process not built-in options. */ + foreach(option, unknown_options) + { + DefElem *defel = lfirst_node(DefElem, option); + bool processed = false; + + if (!is_from) + processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); + if (!processed) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("option \"%s\" not recognized", + defel->defname), + parser_errposition(pstate, defel->location))); + } + list_free(unknown_options); I think we can check the duplicated options in the core as we discussed. --- +static void +CopyToTextBasedInit(CopyToState cstate) +{ +} and +static void +CopyToBinaryInit(CopyToState cstate) +{ +} Do we really need separate callbacks for the same behavior? I think we can have a common init function say CopyToBuitinInit() that does nothing. Or we can make the init callback optional. The same is true for process-option callback. --- List *convert_select; /* list of column names (can be NIL) */ + const CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; I think CopyToStateData is a better place to have CopyToRoutine. copy_data_dest_cb is also there. --- - if (strcmp(fmt, "text") == 0) - /* default format */ ; - else if (strcmp(fmt, "csv") == 0) - opts_out->csv_mode = true; - else if (strcmp(fmt, "binary") == 0) - opts_out->binary = true; + + if (is_from) + { + char *fmt = defGetString(defel); + + if (strcmp(fmt, "text") == 0) + /* default format */ ; + else if (strcmp(fmt, "csv") == 0) + { + opts_out->csv_mode = true; + } + else if (strcmp(fmt, "binary") == 0) + { + opts_out->binary = true; + } else - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt\), - parser_errposition(pstate, defel->location))); + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, defel); The 0002 patch replaces the options checks with ProcessCopyOptionFormatFrom(). However, both ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would set format-related options such as opts_out->csv_mode etc, which seems not elegant. IIUC the reason why we process only the "format" option first is to set the callback functions and call the init callback. So I think we don't necessarily need to do both setting callbacks and setting format-related options together. Probably we can do only the callback stuff first and then set format-related options in the original place we used to do? --- +static void +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf) +{ + int16 format = 0; + int natts = list_length(cstate->attnumlist); + int i; + + pq_sendbyte(buf, format); /* overall format */ + pq_sendint16(buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(buf, format); /* per-column formats */ +} This function and CopyToBinaryFillCopyOutResponse() fill three things: overall format, the number of columns, and per-column formats. While this approach is flexible, extensions will have to understand the format of CopyOutResponse message. An alternative is to have one or more callbacks that return these three things. --- + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(Fmgr\Info)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } Is preparing the out functions an extension's responsibility? I thought the core could prepare them based on the overall format specified by extensions, as long as the overall format matches the actual data format to send. What do you think? --- + /* + * Called when COPY TO via the PostgreSQL protocol is started. This must + * fill buf as a valid CopyOutResponse message: + * + */ + /*-- + * +--------+--------+--------+--------+--------+ +--------+--------+ + * | Format | N attributes | Attr1's format |...| AttrN's format | + * +--------+--------+--------+--------+--------+ +--------+--------+ + * 0: text 0: text 0: text + * 1: binary 1: binary 1: binary + */ I think this kind of diagram could be missed from being updated when we update the CopyOutResponse format. It's better to refer to the documentation instead. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 05:45 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-30 05:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900, Masahiko Sawada <[email protected]> wrote: > --- > + if (!format_specified) > + /* Set the default format. */ > + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); > + > > I think we can pass "text" in this case instead of NULL. That way, > ProcessCopyOptionFormatTo doesn't need to handle NULL case. Yes, we can do it. But it needs a DefElem allocation. Is it acceptable? > We need curly brackets for this "if branch" as follows: > > if (!format_specifed) > { > /* Set the default format. */ > ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); > } Oh, sorry. I assumed that pgindent adjusts the style too. > --- > + /* Process not built-in options. */ > + foreach(option, unknown_options) > + { > + DefElem *defel = lfirst_node(DefElem, option); > + bool processed = false; > + > + if (!is_from) > + processed = > opts_out->to_routine->CopyToProcessOption(cstate, defel); > + if (!processed) > + ereport(ERROR, > + (errcode(ERRCODE_SYNTAX_ERROR), > + errmsg("option \"%s\" not recognized", > + defel->defname), > + parser_errposition(pstate, > defel->location))); > + } > + list_free(unknown_options); > > I think we can check the duplicated options in the core as we discussed. Oh, sorry. I missed the part. I'll implement it. > --- > +static void > +CopyToTextBasedInit(CopyToState cstate) > +{ > +} > > and > > +static void > +CopyToBinaryInit(CopyToState cstate) > +{ > +} > > Do we really need separate callbacks for the same behavior? I think we > can have a common init function say CopyToBuitinInit() that does > nothing. Or we can make the init callback optional. > > The same is true for process-option callback. OK. I'll make them optional. > --- > List *convert_select; /* list of column names (can be NIL) */ > + const CopyToRoutine *to_routine; /* callback > routines for COPY TO */ > } CopyFormatOptions; > > I think CopyToStateData is a better place to have CopyToRoutine. > copy_data_dest_cb is also there. We can do it but ProcessCopyOptions() accepts NULL CopyToState for file_fdw. Can we create an empty CopyToStateData internally like we did for opts_out in ProcessCopyOptions()? (But it requires exporting CopyToStateData. We'll export it in a later patch but it's not yet in 0001.) > The 0002 patch replaces the options checks with > ProcessCopyOptionFormatFrom(). However, both > ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would > set format-related options such as opts_out->csv_mode etc, which seems > not elegant. IIUC the reason why we process only the "format" option > first is to set the callback functions and call the init callback. So > I think we don't necessarily need to do both setting callbacks and > setting format-related options together. Probably we can do only the > callback stuff first and then set format-related options in the > original place we used to do? If we do it, we need to write the (strcmp(format, "csv") == 0) condition in copyto.c and copy.c. I wanted to avoid it. I think that the duplication (setting opts_out->csv_mode in copyto.c and copyfrom.c) is not a problem. But it's not a strong opinion. If (strcmp(format, "csv") == 0) duplication is better than opts_out->csv_mode = true duplication, I'll do it. BTW, if we want to make the CSV format implementation more modularized, we will remove opts_out->csv_mode, move CSV related options to CopyToCSVProcessOption() and keep CSV related options in its opaque space. For example, opts_out->force_quote exists in COPY TO opaque space but doesn't exist in COPY FROM opaque space because it's not used in COPY FROM. > +static void > +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf) > +{ > + int16 format = 0; > + int natts = list_length(cstate->attnumlist); > + int i; > + > + pq_sendbyte(buf, format); /* overall format */ > + pq_sendint16(buf, natts); > + for (i = 0; i < natts; i++) > + pq_sendint16(buf, format); /* per-column formats */ > +} > > This function and CopyToBinaryFillCopyOutResponse() fill three things: > overall format, the number of columns, and per-column formats. While > this approach is flexible, extensions will have to understand the > format of CopyOutResponse message. An alternative is to have one or > more callbacks that return these three things. Yes, we can choose the approach. I don't have a strong opinion on which approach to choose. > + /* Get info about the columns we need to process. */ > + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * > sizeof(Fmgr\Info)); > + foreach(cur, cstate->attnumlist) > + { > + int attnum = lfirst_int(cur); > + Oid out_func_oid; > + bool isvarlena; > + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); > + > + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); > + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); > + } > > Is preparing the out functions an extension's responsibility? I > thought the core could prepare them based on the overall format > specified by extensions, as long as the overall format matches the > actual data format to send. What do you think? Hmm. I want to keep the preparation as an extension's responsibility. Because it's not needed for all formats. For example, Apache Arrow FORMAT doesn't need it. And JSON FORMAT doesn't need it too because it use composite_to_json(). > + /* > + * Called when COPY TO via the PostgreSQL protocol is > started. This must > + * fill buf as a valid CopyOutResponse message: > + * > + */ > + /*-- > + * +--------+--------+--------+--------+--------+ +--------+--------+ > + * | Format | N attributes | Attr1's format |...| AttrN's format | > + * +--------+--------+--------+--------+--------+ +--------+--------+ > + * 0: text 0: text 0: text > + * 1: binary 1: binary 1: binary > + */ > > I think this kind of diagram could be missed from being updated when > we update the CopyOutResponse format. It's better to refer to the > documentation instead. It makes sense. I couldn't find the documentation when I wrote it but I found it now...: https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY Is there recommended comment style to refer a documentation? "See doc/src/sgml/protocol.sgml for the CopyOutResponse message details" is OK? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 07:20 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-30 07:20 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jan 30, 2024 at 02:45:31PM +0900, Sutou Kouhei wrote: > In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900, > Masahiko Sawada <[email protected]> wrote: > >> --- >> + if (!format_specified) >> + /* Set the default format. */ >> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); >> + >> >> I think we can pass "text" in this case instead of NULL. That way, >> ProcessCopyOptionFormatTo doesn't need to handle NULL case. > > Yes, we can do it. But it needs a DefElem allocation. Is it > acceptable? I don't think that there is a need for a DelElem at all here? While I am OK with the choice of calling CopyToInit() in the ProcessCopyOption*() routines that exist to keep the set of callbacks local to copyto.c and copyfrom.c, I think that this should not bother about setting opts_out->csv_mode or opts_out->csv_mode but just set the opts_out->{to,from}_routine callbacks. >> +static void >> +CopyToTextBasedInit(CopyToState cstate) >> +{ >> +} >> >> and >> >> +static void >> +CopyToBinaryInit(CopyToState cstate) >> +{ >> +} >> >> Do we really need separate callbacks for the same behavior? I think we >> can have a common init function say CopyToBuitinInit() that does >> nothing. Or we can make the init callback optional. Keeping empty options does not strike as a bad idea, because this forces extension developers to think about this code path rather than just ignore it. Now, all the Init() callbacks are empty for the in-core callbacks, so I think that we should just remove it entirely for now. Let's keep the core patch a maximum simple. It is always possible to build on top of it depending on what people need. It's been mentioned that JSON would want that, but this also proves that we just don't care about that for all the in-core callbacks, as well. I would choose a minimalistic design for now. >> + /* Get info about the columns we need to process. */ >> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * >> sizeof(Fmgr\Info)); >> + foreach(cur, cstate->attnumlist) >> + { >> + int attnum = lfirst_int(cur); >> + Oid out_func_oid; >> + bool isvarlena; >> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); >> + >> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); >> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); >> + } >> >> Is preparing the out functions an extension's responsibility? I >> thought the core could prepare them based on the overall format >> specified by extensions, as long as the overall format matches the >> actual data format to send. What do you think? > > Hmm. I want to keep the preparation as an extension's > responsibility. Because it's not needed for all formats. For > example, Apache Arrow FORMAT doesn't need it. And JSON > FORMAT doesn't need it too because it use > composite_to_json(). I agree that it could be really useful for extensions to be able to force that. We already know that for the in-core formats we've cared about being able to enforce the way data is handled in input and output. > It makes sense. I couldn't find the documentation when I > wrote it but I found it now...: > https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY > > Is there recommended comment style to refer a documentation? > "See doc/src/sgml/protocol.sgml for the CopyOutResponse > message details" is OK? There are a couple of places in the C code where we refer to SGML docs when it comes to specific details, so using a method like that here to avoid a duplication with the docs sounds sensible for me. I would be really tempted to put my hands on this patch to put into shape a minimal set of changes because I'm caring quite a lot about the performance gains reported with the removal of the "if" checks in the per-row callbacks, and that's one goal of this thread quite independent on the extensibility. Sutou-san, would you be OK with that? -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 08:15 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-30 08:15 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 16:20:54 +0900, Michael Paquier <[email protected]> wrote: >>> + if (!format_specified) >>> + /* Set the default format. */ >>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); >>> + >>> >>> I think we can pass "text" in this case instead of NULL. That way, >>> ProcessCopyOptionFormatTo doesn't need to handle NULL case. >> >> Yes, we can do it. But it needs a DefElem allocation. Is it >> acceptable? > > I don't think that there is a need for a DelElem at all here? We use defel->location for an error message. (We don't need to set location for the default "text" DefElem.) > While I > am OK with the choice of calling CopyToInit() in the > ProcessCopyOption*() routines that exist to keep the set of callbacks > local to copyto.c and copyfrom.c, I think that this should not bother > about setting opts_out->csv_mode or opts_out->csv_mode but just set > the opts_out->{to,from}_routine callbacks. OK. I'll keep opts_out->{csv_mode,binary} in copy.c. > Now, all the Init() callbacks are empty for the > in-core callbacks, so I think that we should just remove it entirely > for now. Let's keep the core patch a maximum simple. It is always > possible to build on top of it depending on what people need. It's > been mentioned that JSON would want that, but this also proves that we > just don't care about that for all the in-core callbacks, as well. I > would choose a minimalistic design for now. OK. Let's remove Init() callbacks from the first patch set. > I would be really tempted to put my hands on this patch to put into > shape a minimal set of changes because I'm caring quite a lot about > the performance gains reported with the removal of the "if" checks in > the per-row callbacks, and that's one goal of this thread quite > independent on the extensibility. Sutou-san, would you be OK with > that? Yes, sure. (We want to focus on the performance gains in the first patch set and then focus on extensibility again, right?) For the purpose, I think that the v7 patch set is more suitable than the v9 patch set. The v7 patch set doesn't include Init() callbacks, custom options validation support or extra Copy{In,Out}Response support. But the v7 patch set misses the removal of the "if" checks in NextCopyFromRawFields() that exists in the v9 patch set. I'm not sure how much performance will improve by this but it may be worth a try. Can I prepare the v10 patch set as "the v7 patch set" + "the removal of the "if" checks in NextCopyFromRawFields()"? (+ reverting opts_out->{csv_mode,binary} changes in ProcessCopyOptions().) Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 08:37 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2024-01-30 08:37 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jan 30, 2024 at 05:15:11PM +0900, Sutou Kouhei wrote: > We use defel->location for an error message. (We don't need > to set location for the default "text" DefElem.) Yeah, but you should not need to have this error in the paths that set the callback routines in opts_out if the same validation happens a few lines before, in copy.c. > Yes, sure. > (We want to focus on the performance gains in the first > patch set and then focus on extensibility again, right?) Yep, exactly, the numbers are too good to just ignore. I don't want to hijack the thread, but I am really worried about the complexities this thread is getting into because we are trying to shape the callbacks in the most generic way possible based on *two* use cases. This is going to be a never-ending discussion. I'd rather get some simple basics, and then we can discuss if tweaking the callbacks is really necessary or not. Even after introducing the pg_proc lookups to get custom callbacks. > For the purpose, I think that the v7 patch set is more > suitable than the v9 patch set. The v7 patch set doesn't > include Init() callbacks, custom options validation support > or extra Copy{In,Out}Response support. But the v7 patch set > misses the removal of the "if" checks in > NextCopyFromRawFields() that exists in the v9 patch set. I'm > not sure how much performance will improve by this but it > may be worth a try. Yeah.. The custom options don't seem like an absolute strong requirement for the first shot with the callbacks or even the possibility to retrieve the callbacks from a function call. I mean, you could provide some control with SET commands and a few GUCs, at least, even if that would be strange. Manipulations with a list of DefElems is the intuitive way to have custom options at query level, but we also have to guess the set of callbacks from this list of DefElems coming from the query. You see my point, I am not sure if it would be the best thing to process twice the options, especially when it comes to decide if a DefElem should be valid or not depending on the callbacks used. Or we could use a kind of "special" DefElem where we could store a set of key:value fed to a callback :) > Can I prepare the v10 patch set as "the v7 patch set" + "the > removal of the "if" checks in NextCopyFromRawFields()"? > (+ reverting opts_out->{csv_mode,binary} changes in > ProcessCopyOptions().) Yep, if I got it that would make sense to me. If you can do that, that would help quite a bit. :) -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-31 05:11 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2024-01-31 05:11 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 17:37:35 +0900, Michael Paquier <[email protected]> wrote: >> We use defel->location for an error message. (We don't need >> to set location for the default "text" DefElem.) > > Yeah, but you should not need to have this error in the paths that set > the callback routines in opts_out if the same validation happens a few > lines before, in copy.c. Ah, yes. defel->location is used in later patches. For example, it's used when a COPY handler for the specified FORMAT isn't found. > I am really worried about the complexities > this thread is getting into because we are trying to shape the > callbacks in the most generic way possible based on *two* use cases. > This is going to be a never-ending discussion. I'd rather get some > simple basics, and then we can discuss if tweaking the callbacks is > really necessary or not. Even after introducing the pg_proc lookups > to get custom callbacks. I understand your concern. Let's introduce minimal callbacks as the first step. I think that we completed our design discussion for this feature. We can choose minimal callbacks based on the discussion. > The custom options don't seem like an absolute strong > requirement for the first shot with the callbacks or even the > possibility to retrieve the callbacks from a function call. I mean, > you could provide some control with SET commands and a few GUCs, at > least, even if that would be strange. Manipulations with a list of > DefElems is the intuitive way to have custom options at query level, > but we also have to guess the set of callbacks from this list of > DefElems coming from the query. You see my point, I am not sure > if it would be the best thing to process twice the options, especially > when it comes to decide if a DefElem should be valid or not depending > on the callbacks used. Or we could use a kind of "special" DefElem > where we could store a set of key:value fed to a callback :) Interesting. Let's remove custom options support from the initial minimal callbacks. >> Can I prepare the v10 patch set as "the v7 patch set" + "the >> removal of the "if" checks in NextCopyFromRawFields()"? >> (+ reverting opts_out->{csv_mode,binary} changes in >> ProcessCopyOptions().) > > Yep, if I got it that would make sense to me. If you can do that, > that would help quite a bit. :) I've prepared the v10 patch set. Could you try this? Changes since the v7 patch set: 0001: * Remove CopyToProcessOption() callback * Remove CopyToGetFormat() callback * Revert passing CopyToState to ProcessCopyOptions() * Revert moving "opts_out->{csv_mode,binary} = true" to ProcessCopyOptionFormatTo() * Change to receive "const char *format" instead "DefElem *defel" by ProcessCopyOptionFormatTo() 0002: * Remove CopyFromProcessOption() callback * Remove CopyFromGetFormat() callback * Change to receive "const char *format" instead "DefElem *defel" by ProcessCopyOptionFormatFrom() * Remove "if (cstate->opts.csv_mode)" branches from NextCopyFromRawFields() FYI: Here are Copy{From,To}Routine in the v10 patch set. I think that only Copy{From,To}OneRow are minimal callbacks for the performance gain. But can we keep Copy{From,To}Start and Copy{From,To}End for consistency? We can remove a few {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It doesn't depend on the number of COPY target tuples. So they will not affect performance. /* Routines for a COPY FROM format implementation. */ typedef struct CopyFromRoutine { /* * Called when COPY FROM is started. This will initialize something and * receive a header. */ void (*CopyFromStart) (CopyFromState cstate, TupleDesc tupDesc); /* Copy one row. It returns false if no more tuples. */ bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); /* Called when COPY FROM is ended. This will finalize something. */ void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; /* Routines for a COPY TO format implementation. */ typedef struct CopyToRoutine { /* Called when COPY TO is started. This will send a header. */ void (*CopyToStart) (CopyToState cstate, TupleDesc tupDesc); /* Copy one row for COPY TO. */ void (*CopyToOneRow) (CopyToState cstate, TupleTableSlot *slot); /* Called when COPY TO is ended. This will send a trailer. */ void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-31 05:39 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Michael Paquier @ 2024-01-31 05:39 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 31, 2024 at 02:11:22PM +0900, Sutou Kouhei wrote: > Ah, yes. defel->location is used in later patches. For > example, it's used when a COPY handler for the specified > FORMAT isn't found. I see. > I've prepared the v10 patch set. Could you try this? Thanks, I'm looking into that now. > FYI: Here are Copy{From,To}Routine in the v10 patch set. I > think that only Copy{From,To}OneRow are minimal callbacks > for the performance gain. But can we keep Copy{From,To}Start > and Copy{From,To}End for consistency? We can remove a few > {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It > doesn't depend on the number of COPY target tuples. So they > will not affect performance. I think I'm OK to keep the start/end callbacks. This makes the code more consistent as a whole, as well. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-17 20:50 Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-03-17 20:50 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Mar 4, 2025 at 4:06 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoAwOP7p6LgmkPGqPuJ5KbJPPQsSZsFzwCDguwzr9F677Q@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 3 Mar 2025 11:06:39 -0800, > Masahiko Sawada <[email protected]> wrote: > > > I agree with the fix and the patch looks good to me. I've updated the > > commit message and am going to push, barring any objections. > > Thanks! > > I've rebased the patch set. Here is a summary again: Thank you for updating the patches. Here are some review comments on the 0001 patch: + if (strcmp(format, "text") == 0) + { + /* "csv_mode == false && binary == false" means "text" */ + return; + } + else if (strcmp(format, "csv") == 0) + { + opts_out->csv_mode = true; + return; + } + else if (strcmp(format, "binary") == 0) + { + opts_out->binary = true; + return; + } + + /* custom format */ + if (!is_from) + { + funcargtypes[0] = INTERNALOID; + handlerOid = LookupFuncName(list_make1(makeString(format)), 1, + funcargtypes, true); + } + if (!OidIsValid(handlerOid)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", format), + parser_errposition(pstate, defel->location))); I think that built-in formats also need to have their handler functions. This seems to be a conventional way for customizable features such as tablesample and access methods, and we can simplify this function. --- I think we need to update the documentation to describe how users can define the handler functions and what each callback function is responsible for. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-19 02:56 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-03-19 02:56 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Mar 2025 13:50:03 -0700, Masahiko Sawada <[email protected]> wrote: > I think that built-in formats also need to have their handler > functions. This seems to be a conventional way for customizable > features such as tablesample and access methods, and we can simplify > this function. OK. 0008 in the attached v37 patch set does it. > I think we need to update the documentation to describe how users can > define the handler functions and what each callback function is > responsible for. I agree with it but we haven't finalized public APIs yet. Can we defer it after we finalize public APIs? (Proposed public APIs exist in 0003, 0006 and 0007.) And could someone help (take over if possible) writing a document for this feature? I'm not good at writing a document in English... 0009 in the attached v37 patch set has a draft of it. It's based on existing documents in doc/src/sgml/ and *.h. 0001-0007 aren't changed from v36 patch set. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-20 00:49 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-03-20 00:49 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote: > And could someone help (take over if possible) writing a > document for this feature? I'm not good at writing a > document in English... 0009 in the attached v37 patch set > has a draft of it. It's based on existing documents in > doc/src/sgml/ and *.h. > > I haven't touched the innards of the structs aside from changing programlisting to synopsis. And redoing the two section opening paragraphs to better integrate with the content in the chapter opening. The rest I kinda went to town on... David J. diff --git a/doc/src/sgml/copy-handler.sgml b/doc/src/sgml/copy-handler.sgml index f602debae6..9d2897a104 100644 --- a/doc/src/sgml/copy-handler.sgml +++ b/doc/src/sgml/copy-handler.sgml @@ -10,56 +10,72 @@ <para> <productname>PostgreSQL</productname> supports custom <link linkend="sql-copy"><literal>COPY</literal></link> - handlers. The <literal>COPY</literal> handlers can use different copy format - instead of built-in <literal>text</literal>, <literal>csv</literal> - and <literal>binary</literal>. + handlers; adding additional <replaceable>format_name</replaceable> options + to the <literal>FORMAT</literal> clause. </para> <para> - At the SQL level, a table sampling method is represented by a single SQL - function, typically implemented in C, having the signature -<programlisting> -format_name(internal) RETURNS copy_handler -</programlisting> - The name of the function is the same name appearing in - the <literal>FORMAT</literal> option. The <type>internal</type> argument is - a dummy that simply serves to prevent this function from being called - directly from an SQL command. The real argument is <literal>bool - is_from</literal>. If the handler is used by <literal>COPY FROM</literal>, - it's <literal>true</literal>. If the handler is used by <literal>COPY - FROM</literal>, it's <literal>false</literal>. + At the SQL level, a copy handler method is represented by a single SQL + function (see <xref linkend="sql-createfunction"/>), typically implemented in + C, having the signature +<synopsis> +<replaceable>format_name</replaceable>(internal) RETURNS <literal>copy_handler</literal> +</synopsis> + The function's name is then accepted as a valid <replaceable>format_name</replaceable>. + The return pseudo-type <literal>copy_handler</literal> informs the system that + this function needs to be registered as a copy handler. + The <type>internal</type> argument is a dummy that prevents + this function from being called directly from an SQL command. As the + handler implementation must be server-lifetime immutable; this SQL function's + volatility should be marked immutable. The <literal>link_symbol</literal> + for this function is the name of the implementation function, described next. </para> <para> - The function must return <type>CopyFromRoutine *</type> when - the <literal>is_from</literal> argument is <literal>true</literal>. - The function must return <type>CopyToRoutine *</type> when - the <literal>is_from</literal> argument is <literal>false</literal>. + The implementation function signature expected for the function named + in the <literal>link_symbol</literal> is: +<synopsis> +Datum +<replaceable>copy_format_handler</replaceable>(PG_FUNCTION_ARGS) +</synopsis> + The convention for the name is to replace the word + <replaceable>format</replaceable> in the placeholder above with the value given + to <replaceable>format_name</replaceable> in the SQL function. + The first argument is a <type>boolean</type> that indicates whether the handler + must provide a pointer to its implementation for <literal>COPY FROM</literal> + (a <type>CopyFromRoutine *</type>). If <literal>false</literal>, the handler + must provide a pointer to its implementation of <literal>COPY TO</literal> + (a <type>CopyToRoutine *</type>). These structs are declared in + <filename>src/include/commands/copyapi.h</filename>. </para> <para> - The <type>CopyFromRoutine</type> and <type>CopyToRoutine</type> struct types - are declared in <filename>src/include/commands/copyapi.h</filename>, - which see for additional details. + The structs hold pointers to implementation functions for + initializing, starting, processing rows, and ending a copy operation. + The specific structures vary a bit between <literal>COPY FROM</literal> and + <literal>COPY TO</literal> so the next two sections describes each + in detail. </para> <sect1 id="copy-handler-from"> <title>Copy From Handler</title> <para> - The <literal>COPY</literal> handler function for <literal>COPY - FROM</literal> returns a <type>CopyFromRoutine</type> struct containing - pointers to the functions described below. All functions are required. + The opening to this chapter describes how the executor will call the + main handler function with, in this case, + a <type>boolean</type> <literal>true</literal>, and expect to receive a + <type>CopyFromRoutine *</type> <type>Datum</type>. This section describes + the components of the <type>CopyFromRoutine</type> struct. </para> <para> -<programlisting> +<synopsis> void CopyFromInFunc(CopyFromState cstate, Oid atttypid, FmgrInfo *finfo, Oid *typioparam); -</programlisting> +</synopsis> This sets input function information for the given <literal>atttypid</literal> attribute. This function is called once @@ -110,11 +126,11 @@ CopyFromInFunc(CopyFromState cstate, </para> <para> -<programlisting> +<synopsis> void CopyFromStart(CopyFromState cstate, TupleDesc tupDesc); -</programlisting> +</synopsis> This starts a <literal>COPY FROM</literal>. This function is called once at the beginning of <literal>COPY FROM</literal>. @@ -144,13 +160,13 @@ CopyFromStart(CopyFromState cstate, </para> <para> -<programlisting> +<synopsis> bool CopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); -</programlisting> +</synopsis> This reads one row from the source and fill <literal>values</literal> and <literal>nulls</literal>. If there is one or more tuples to be read, @@ -202,10 +218,10 @@ CopyFromOneRow(CopyFromState cstate, </para> <para> -<programlisting> +<synopsis> void CopyFromEnd(CopyFromState cstate); -</programlisting> +</synopsis> This ends a <literal>COPY FROM</literal>. This function is called once at the end of <literal>COPY FROM</literal>. @@ -232,18 +248,20 @@ CopyFromEnd(CopyFromState cstate); <title>Copy To Handler</title> <para> - The <literal>COPY</literal> handler function for <literal>COPY - TO</literal> returns a <type>CopyToRoutine</type> struct containing - pointers to the functions described below. All functions are required. + The opening to this chapter describes how the executor will call the + main handler function with, in this case, + a <type>boolean</type> <literal>false</literal>, and expect to receive a + <type>CopyInRoutine *</type> <type>Datum</type>. This section describes + the components of the <type>CopyInRoutine</type> struct. </para> <para> -<programlisting> +<synopsis> void CopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo); -</programlisting> +</synopsis> This sets output function information for the given <literal>atttypid</literal> attribute. This function is called once @@ -284,11 +302,11 @@ CopyToOutFunc(CopyToState cstate, </para> <para> -<programlisting> +<synopsis> void CopyToStart(CopyToState cstate, TupleDesc tupDesc); -</programlisting> +</synopsis> This starts a <literal>COPY TO</literal>. This function is called once at the beginning of <literal>COPY TO</literal>. @@ -316,11 +334,11 @@ CopyToStart(CopyToState cstate, </para> <para> -<programlisting> +<synopsis> bool CopyToOneRow(CopyToState cstate, TupleTableSlot *slot); -</programlisting> +</synopsis> This writes one row stored in <literal>slot</literal> to the destination. @@ -347,10 +365,10 @@ CopyToOneRow(CopyToState cstate, </para> <para> -<programlisting> +<synopsis> void CopyToEnd(CopyToState cstate); -</programlisting> +</synopsis> This ends a <literal>COPY TO</literal>. This function is called once at the end of <literal>COPY TO</literal>. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-20 01:24 Sutou Kouhei <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-03-20 01:24 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700, "David G. Johnston" <[email protected]> wrote: >> And could someone help (take over if possible) writing a >> document for this feature? I'm not good at writing a >> document in English... 0009 in the attached v37 patch set >> has a draft of it. It's based on existing documents in >> doc/src/sgml/ and *.h. >> >> > I haven't touched the innards of the structs aside from changing > programlisting to synopsis. And redoing the two section opening paragraphs > to better integrate with the content in the chapter opening. > > The rest I kinda went to town on... Thanks!!! It's very helpful!!! I've applied your patch. 0009 is only changed. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-22 00:31 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-03-22 00:31 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote: > Hi, > > In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format > implementations" on Mon, 17 Mar 2025 13:50:03 -0700, > Masahiko Sawada <[email protected]> wrote: > > > I think that built-in formats also need to have their handler > > functions. This seems to be a conventional way for customizable > > features such as tablesample and access methods, and we can simplify > > this function. > > OK. 0008 in the attached v37 patch set does it. > > tl/dr; We need to exclude from our SQL function search any function that doesn't declare copy_handler as its return type. ("function text must return type copy_handler" is not an acceptable error message) We need to accept identifiers in FORMAT and parse the optional catalog, schema, and object name portions. (Restrict our function search to the named schema if provided.) Detail: Fun thing...(not sure how much of this is covered above: I do see, but didn't scour, the security discussion): -- create some poison create function public.text(internal) returns boolean language c as '/home/davidj/gotya/gotya', 'gotit'; CREATE FUNCTION -- inject it postgres=# set search_path to public,pg_catalog; SET -- watch it die postgres=# copy (select 1) to stdout (format text); ERROR: function text must return type copy_handler LINE 1: copy (select 1) to stdout (format text); I'm especially concerned about extensions here. We shouldn't be locating any SQL function that doesn't have a copy_handler return type. Unfortunately, LookupFuncName seems incapable of doing what we want here. I suggest we create a new lookup routine where we can specify the return argument type as a required element. That would cleanly mitigate the denial-of-service attack/accident vector demonstrated above (the text returning function should have zero impact on how this feature behaves). If someone does create a handler SQL function without using copy_handler return type we'd end up showing "COPY format 'david' not recognized" - a developer should be able to figure out they didn't put a correct return type on their handler function and that is why the system did not register it. A second concern is simply people wanting to name things the same; or, why namespaces were invented. Can we just accept a proper identifier after FORMAT so we can use schema-qualified names? (FORMAT "davescopyformat"."david") We can special case the internal schema-less names and internally force pg_catalog to avoid them being shadowed. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-22 05:07 Masahiko Sawada <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-03-22 05:07 UTC (permalink / raw) To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Mar 21, 2025 at 5:32 PM David G. Johnston <[email protected]> wrote: > > On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote: >> >> Hi, >> >> In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com> >> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Mar 2025 13:50:03 -0700, >> Masahiko Sawada <[email protected]> wrote: >> >> > I think that built-in formats also need to have their handler >> > functions. This seems to be a conventional way for customizable >> > features such as tablesample and access methods, and we can simplify >> > this function. >> >> OK. 0008 in the attached v37 patch set does it. >> > > tl/dr; > > We need to exclude from our SQL function search any function that doesn't declare copy_handler as its return type. > ("function text must return type copy_handler" is not an acceptable error message) > > We need to accept identifiers in FORMAT and parse the optional catalog, schema, and object name portions. > (Restrict our function search to the named schema if provided.) > > Detail: > > Fun thing...(not sure how much of this is covered above: I do see, but didn't scour, the security discussion): > > -- create some poison > create function public.text(internal) returns boolean language c as '/home/davidj/gotya/gotya', 'gotit'; > CREATE FUNCTION > > -- inject it > postgres=# set search_path to public,pg_catalog; > SET > > -- watch it die > postgres=# copy (select 1) to stdout (format text); > ERROR: function text must return type copy_handler > LINE 1: copy (select 1) to stdout (format text); > > I'm especially concerned about extensions here. > > We shouldn't be locating any SQL function that doesn't have a copy_handler return type. Unfortunately, LookupFuncName seems incapable of doing what we want here. I suggest we create a new lookup routine where we can specify the return argument type as a required element. That would cleanly mitigate the denial-of-service attack/accident vector demonstrated above (the text returning function should have zero impact on how this feature behaves). If someone does create a handler SQL function without using copy_handler return type we'd end up showing "COPY format 'david' not recognized" - a developer should be able to figure out they didn't put a correct return type on their handler function and that is why the system did not register it. Just to be clear, the patch checks the function's return type before calling it: funcargtypes[0] = INTERNALOID; handlerOid = LookupFuncName(list_make1(makeString(format)), 1, funcargtypes, true); if (!OidIsValid(handlerOid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY format \"%s\" not recognized", format), parser_errposition(pstate, defel->location))); /* check that handler has correct return type */ if (get_func_rettype(handlerOid) != COPY_HANDLEROID) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("function %s must return type %s", format, "copy_handler"), parser_errposition(pstate, defel->location))); So would changing the error message to like "COPY format 'text' not recognized" untangle your concern? FYI the same is true for TABLESAMPLE; it invokes a function with the specified method name and checks the returned Node type: =# select * from pg_class tablesample text (0); ERROR: function text must return type tsm_handler A difference between TABLESAMPLE and COPY format is that the former accepts a qualified name but the latter doesn't: =# create extension tsm_system_rows ; =# create schema s1; =# create function s1.system_rows(internal) returns void language c as 'tsm_system_rows.so', 'tsm_system_rows_handler'; =# \df *.system_rows List of functions Schema | Name | Result data type | Argument data types | Type --------+-------------+------------------+---------------------+------ public | system_rows | tsm_handler | internal | func s1 | system_rows | void | internal | func (2 rows) postgres(1:1194923)=# select count(*) from pg_class tablesample system_rows(0); count ------- 0 (1 row) postgres(1:1194923)=# select count(*) from pg_class tablesample s1.system_rows(0); ERROR: function s1.system_rows must return type tsm_handler > A second concern is simply people wanting to name things the same; or, why namespaces were invented. Yeah, I think that the custom COPY format should support qualified names at least. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-22 05:23 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-03-22 05:23 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Friday, March 21, 2025, Masahiko Sawada <[email protected]> wrote: > On Fri, Mar 21, 2025 at 5:32 PM David G. Johnston > <[email protected]> wrote: > > > > On Tue, Mar 18, 2025 at 7:56 PM Sutou Kouhei <[email protected]> wrote: > >> > >> Hi, > >> > >> In <CAD21AoDU=bYRDDY8MzCXAfg4h9XTeTBdM-wVJaO1t4UcseCpuA@mail.gmail.com> > >> "Re: Make COPY format extendable: Extract COPY TO format > implementations" on Mon, 17 Mar 2025 13:50:03 -0700, > >> Masahiko Sawada <[email protected]> wrote: > >> > >> > I think that built-in formats also need to have their handler > >> > functions. This seems to be a conventional way for customizable > >> > features such as tablesample and access methods, and we can simplify > >> > this function. > >> > >> OK. 0008 in the attached v37 patch set does it. > >> > > > > tl/dr; > > > > We need to exclude from our SQL function search any function that > doesn't declare copy_handler as its return type. > > ("function text must return type copy_handler" is not an acceptable > error message) > > > > We need to accept identifiers in FORMAT and parse the optional catalog, > schema, and object name portions. > > (Restrict our function search to the named schema if provided.) > > > > Detail: > > > > Fun thing...(not sure how much of this is covered above: I do see, but > didn't scour, the security discussion): > > > > -- create some poison > > create function public.text(internal) returns boolean language c as > '/home/davidj/gotya/gotya', 'gotit'; > > CREATE FUNCTION > > > > -- inject it > > postgres=# set search_path to public,pg_catalog; > > SET > > > > -- watch it die > > postgres=# copy (select 1) to stdout (format text); > > ERROR: function text must return type copy_handler > > LINE 1: copy (select 1) to stdout (format text); > > > > I'm especially concerned about extensions here. > > > > We shouldn't be locating any SQL function that doesn't have a > copy_handler return type. Unfortunately, LookupFuncName seems incapable of > doing what we want here. I suggest we create a new lookup routine where we > can specify the return argument type as a required element. That would > cleanly mitigate the denial-of-service attack/accident vector demonstrated > above (the text returning function should have zero impact on how this > feature behaves). If someone does create a handler SQL function without > using copy_handler return type we'd end up showing "COPY format 'david' not > recognized" - a developer should be able to figure out they didn't put a > correct return type on their handler function and that is why the system > did not register it. > > Just to be clear, the patch checks the function's return type before > calling it: > > funcargtypes[0] = INTERNALOID; > handlerOid = LookupFuncName(list_make1(makeString(format)), 1, > > funcargtypes, true); > if (!OidIsValid(handlerOid)) > ereport(ERROR, > (errcode(ERRCODE_INVALID_PARAMETER_VALUE), > errmsg("COPY format \"%s\" not > recognized", format), > parser_errposition(pstate, > defel->location))); > > /* check that handler has correct return type */ > if (get_func_rettype(handlerOid) != COPY_HANDLEROID) > ereport(ERROR, > (errcode(ERRCODE_WRONG_OBJECT_TYPE), > errmsg("function %s must return type %s", > format, "copy_handler"), > parser_errposition(pstate, > defel->location))); > > So would changing the error message to like "COPY format 'text' not > recognized" untangle your concern? In my example above copy should not fail at all. The text function created in public that returns Boolean would never be seen and the real one in pg_catalog would then be found and behave as expected. > > FYI the same is true for TABLESAMPLE; it invokes a function with the > specified method name and checks the returned Node type: > > =# select * from pg_class tablesample text (0); > ERROR: function text must return type tsm_handler Then this would benefit from the new function I suggest creating since it apparently has the same, IMO, bug. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-22 17:11 David G. Johnston <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-22 17:11 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Fri, Mar 21, 2025 at 10:23 PM David G. Johnston < [email protected]> wrote: > Then this would benefit from the new function I suggest creating since it > apparently has the same, IMO, bug. > > Concretely like I posted here: https://www.postgresql.org/message-id/[email protected]... David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-23 09:01 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-03-23 09:01 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Mar 19, 2025 at 6:25 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700, > "David G. Johnston" <[email protected]> wrote: > > >> And could someone help (take over if possible) writing a > >> document for this feature? I'm not good at writing a > >> document in English... 0009 in the attached v37 patch set > >> has a draft of it. It's based on existing documents in > >> doc/src/sgml/ and *.h. > >> > >> > > I haven't touched the innards of the structs aside from changing > > programlisting to synopsis. And redoing the two section opening paragraphs > > to better integrate with the content in the chapter opening. > > > > The rest I kinda went to town on... > > Thanks!!! It's very helpful!!! > > I've applied your patch. 0009 is only changed. Thank you for updating the patches. I've reviewed the main part of supporting the custom COPY format. Here are some random comments: --- +/* + * Process the "format" option. + * + * This function checks whether the option value is a built-in format such as + * "text" and "csv" or not. If the option value isn't a built-in format, this + * function finds a COPY format handler that returns a CopyToRoutine (for + * is_from == false) or CopyFromRountine (for is_from == true). If no COPY + * format handler is found, this function reports an error. + */ I think this comment needs to be updated as the part "If the option value isn't ..." is no longer true. I think we don't necessarily need to create a separate function ProcessCopyOptionFormat for processing the format option. We need more regression tests for handling the given format name. For example, - more various input patterns. - a function with the specified format name exists but it returns an unexpected Node. - looking for a handler function in a different namespace. etc. --- I think that we should accept qualified names too as the format name like tablesample does. That way, different extensions implementing the same format can be used. --- + if (routine == NULL || !IsA(routine, CopyFromRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%u did not return " + "CopyFromRoutine struct", + opts->handler))); It's not conventional to put a new line between 'ereport(' and 'ERROR' (similarly between 'errcode(' and 'ERRCODE_...'. Also, we don't need to split the error message into multiple lines as it's not long. --- + if (routine == NULL || !IsA(routine, CopyToRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%u did not return " + "CopyToRoutine struct", + opts->handler))); Same as the above comment. --- + descr => 'pseudo-type for the result of a copy to/from method function', s/method function/format function/ --- + Oid handler; /* handler function for custom format routine */ 'handler' is used also for built-in formats. --- +static void +CopyFromInFunc(CopyFromState cstate, Oid atttypid, + FmgrInfo *finfo, Oid *typioparam) +{ + ereport(NOTICE, (errmsg("CopyFromInFunc: atttypid=%d", atttypid))); +} OIDs could be changed across major versions even for built-in types. I think it's better to avoid using it for tests. --- +static void +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", slot->tts_nvalid))); +} Similar to the above comment, the field name 'tts_nvalid' might also be changed in the future, let's use another name. --- +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { + .type = T_CopyFromRoutine, + .CopyFromInFunc = CopyFromInFunc, + .CopyFromStart = CopyFromStart, + .CopyFromOneRow = CopyFromOneRow, + .CopyFromEnd = CopyFromEnd, +}; I'd suggest not using the same function names as the fields. --- +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} Is there any reason to use a different name for public functions? --- +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + The same as the above comment. --- + /* For custom format implementation */ + void *opaque; /* private space */ How about renaming 'private'? --- I've not reviewed the documentation patch yet but I think the patch seems to miss the updates to the description of the FORMAT option in the COPY command section. --- I think we can reorganize the patch set as follows: 1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. 2. Support custom format for both COPY TO and COPY FROM. 3. Expose necessary helper functions such as CopySendEndOfRow(). 4. Add CopyFromSkipErrorRow(). 5. Documentation. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-25 00:45 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Masahiko Sawada @ 2025-03-25 00:45 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Mar 19, 2025 at 6:25 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAKFQuwaMAFMHqxDXR=SxA0mDjdmntrwxZd2w=nSruLNFH-OzLw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 19 Mar 2025 17:49:49 -0700, > "David G. Johnston" <[email protected]> wrote: > > >> And could someone help (take over if possible) writing a > >> document for this feature? I'm not good at writing a > >> document in English... 0009 in the attached v37 patch set > >> has a draft of it. It's based on existing documents in > >> doc/src/sgml/ and *.h. > >> > >> > > I haven't touched the innards of the structs aside from changing > > programlisting to synopsis. And redoing the two section opening paragraphs > > to better integrate with the content in the chapter opening. > > > > The rest I kinda went to town on... > > Thanks!!! It's very helpful!!! > > I've applied your patch. 0009 is only changed. FYI I've implemented an extension to add JSON Lines format as a custom COPY format[1] to check the usability of the COPY format APIs. I think that the exposed APIs are fairly simple and minimum. I didn't find the deficiency and excess of exposed APIs for helping extensions but I find that it would be better to describe what the one-row callback should do to utilize the abstracted destination. For example, in order to use CopyToStateFlush() to write out to the destination, extensions should write the data to cstate->fe_msgbuf. We expose CopyToStateFlush() but not for any functions to write data there such as CopySendString(). It was a bit inconvenient to me but I managed to write the data directly there by #include'ing copyto_internal.h. Regards, [1] https://github.com/MasahikoSawada/pg_copy_jsonlines -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-27 03:28 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 4 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-03-27 03:28 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoAfWrjpTDJ0garVUoXY0WC3Ud4Cu51q+ccWiotm1uo_2A@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sun, 23 Mar 2025 02:01:59 -0700, Masahiko Sawada <[email protected]> wrote: > --- > +/* > + * Process the "format" option. > + * > + * This function checks whether the option value is a built-in format such as > + * "text" and "csv" or not. If the option value isn't a built-in format, this > + * function finds a COPY format handler that returns a CopyToRoutine (for > + * is_from == false) or CopyFromRountine (for is_from == true). If no COPY > + * format handler is found, this function reports an error. > + */ > > I think this comment needs to be updated as the part "If the option > value isn't ..." is no longer true. > > I think we don't necessarily need to create a separate function > ProcessCopyOptionFormat for processing the format option. Hmm. I think that this separated function will increase readability by reducing indentation. But I've removed the separation as you suggested. So the comment is also removed entirely. 0002 includes this. > We need more regression tests for handling the given format name. For example, > > - more various input patterns. > - a function with the specified format name exists but it returns an > unexpected Node. > - looking for a handler function in a different namespace. > etc. I've added the following tests: * Wrong input type handler without namespace * Wrong input type handler with namespace * Wrong return type handler without namespace * Wrong return type handler with namespace * Wrong return value (Copy*Routine isn't returned) handler without namespace * Wrong return value (Copy*Routine isn't returned) handler with namespace * Nonexistent handler * Invalid qualified name * Valid handler without namespace and without search_path * Valid handler without namespace and with search_path * Valid handler with namespace 0002 also includes this. > I think that we should accept qualified names too as the format name > like tablesample does. That way, different extensions implementing the > same format can be used. Implemented. It's implemented after parsing SQL. Is it OK? (It seems that tablesample does it in parsing SQL.) Because "WITH (FORMAT XXX)" is processed as a generic option in gram.y. All generic options are processed as strings. So I keep this. Syntax is "COPY ... WITH (FORMAT 'NAMESPACE.HANDLER_NAME')" not "COPY ... WITH (FORMAT 'NAMESPACE'.'HANDLER_NAME')" because of this choice. 0002 also includes this. > --- > + if (routine == NULL || !IsA(routine, CopyFromRoutine)) > + ereport( > + ERROR, > + (errcode( > + > ERRCODE_INVALID_PARAMETER_VALUE), > + errmsg("COPY handler function " > + "%u did not return " > + "CopyFromRoutine struct", > + opts->handler))); > > It's not conventional to put a new line between 'ereport(' and 'ERROR' > (similarly between 'errcode(' and 'ERRCODE_...'. Also, we don't need > to split the error message into multiple lines as it's not long. Oh, sorry. I can't remember why I used this... I think I trusted pgindent... > --- > + descr => 'pseudo-type for the result of a copy to/from method function', > > s/method function/format function/ Good catch. I used "handler function" not "format function" because we use "handler" in other places. > --- > + Oid handler; /* handler > function for custom format routine */ > > 'handler' is used also for built-in formats. Updated in 0004. > --- > +static void > +CopyFromInFunc(CopyFromState cstate, Oid atttypid, > + FmgrInfo *finfo, Oid *typioparam) > +{ > + ereport(NOTICE, (errmsg("CopyFromInFunc: atttypid=%d", atttypid))); > +} > > OIDs could be changed across major versions even for built-in types. I > think it's better to avoid using it for tests. Oh, I didn't know it. I've changed to use type name instead of OID. It'll be more stable than OID. > --- > +static void > +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot) > +{ > + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", > slot->tts_nvalid))); > +} > > Similar to the above comment, the field name 'tts_nvalid' might also > be changed in the future, let's use another name. Hmm. If the field name is changed, we need to change this code. So changing tests too isn't strange. Anyway, I used more generic text. > --- > +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { > + .type = T_CopyFromRoutine, > + .CopyFromInFunc = CopyFromInFunc, > + .CopyFromStart = CopyFromStart, > + .CopyFromOneRow = CopyFromOneRow, > + .CopyFromEnd = CopyFromEnd, > +}; > > I'd suggest not using the same function names as the fields. OK. I've added "Test" prefix. > --- > +/* > + * Export CopySendEndOfRow() for extensions. We want to keep > + * CopySendEndOfRow() as a static function for > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a > + * compiler. > + */ > +void > +CopyToStateFlush(CopyToState cstate) > +{ > + CopySendEndOfRow(cstate); > +} > > Is there any reason to use a different name for public functions? In this patch set, I use "CopyFrom"/"CopyTo" prefixes for public APIs for custom COPY FORMAT handler extensions. It will help understanding related APIs. Is it strange in PostgreSQL? > --- > + /* For custom format implementation */ > + void *opaque; /* private space */ > > How about renaming 'private'? We should not use "private" because it's a keyword in C++. If we use "private" here, we can't include this file from C++ code. > --- > I've not reviewed the documentation patch yet but I think the patch > seems to miss the updates to the description of the FORMAT option in > the COPY command section. I defer this for now. We can revisit the last documentation patch after we finalize our API. (Or could someone help us?) > I think we can reorganize the patch set as follows: > > 1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and > COPY_DEST_XXX accordingly. > 2. Support custom format for both COPY TO and COPY FROM. > 3. Expose necessary helper functions such as CopySendEndOfRow(). > 4. Add CopyFromSkipErrorRow(). > 5. Documentation. The attached v39 patch set uses the followings: 0001: Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. (Same as 1. in your suggestion) 0002: Support custom format for both COPY TO and COPY FROM. (Same as 2. in your suggestion) 0003: Expose necessary helper functions such as CopySendEndOfRow() and add CopyFromSkipErrorRow(). (3. + 4. in your suggestion) 0004: Define handler functions for built-in formats. (Not included in your suggestion) 0005: Documentation. (WIP) (Same as 5. in your suggestion) We can merge 0001 quickly, right? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-29 05:37 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 3 siblings, 2 replies; 127+ messages in thread From: Masahiko Sawada @ 2025-03-29 05:37 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote: > > > We need more regression tests for handling the given format name. For example, > > > > - more various input patterns. > > - a function with the specified format name exists but it returns an > > unexpected Node. > > - looking for a handler function in a different namespace. > > etc. > > I've added the following tests: > > * Wrong input type handler without namespace > * Wrong input type handler with namespace > * Wrong return type handler without namespace > * Wrong return type handler with namespace > * Wrong return value (Copy*Routine isn't returned) handler without namespace > * Wrong return value (Copy*Routine isn't returned) handler with namespace > * Nonexistent handler > * Invalid qualified name > * Valid handler without namespace and without search_path > * Valid handler without namespace and with search_path > * Valid handler with namespace Probably we can merge these newly added four files into one .sql file? Also we need to add some comments to describe what these queries test. For example, it's not clear to me at a glance what queries in no-schema.sql are going to test as there is no comment there. > > 0002 also includes this. > > > I think that we should accept qualified names too as the format name > > like tablesample does. That way, different extensions implementing the > > same format can be used. > > Implemented. It's implemented after parsing SQL. Is it OK? > (It seems that tablesample does it in parsing SQL.) I think it's okay. One problem in the following chunk I can see is: + qualified_format = stringToQualifiedNameList(format, NULL); + DeconstructQualifiedName(qualified_format, &schema, &fmt); + if (!schema || strcmp(schema, "pg_catalog") == 0) + { + if (strcmp(fmt, "csv") == 0) + opts_out->csv_mode = true; + else if (strcmp(fmt, "binary") == 0) + opts_out->binary = true; + } Non-qualified names depend on the search_path value so it's not necessarily a built-in format. If the user specifies 'csv' with seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily sets csv_mode true. I think we can instead check if the retrieved handler function's OID matches the built-in formats' ones. Also, it's weired to me that cstate has csv_mode and binary fields even though the format should have already been known by the callback functions. Regarding the documentation for the existing options, it says "... only when not using XXX format." some places, where XXX can be replaced with binary or CSV. Once we support custom formats, 'non-CSV mode' would actually include custom formats as well, so we need to update the description too. > > > --- > > +static void > > +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot) > > +{ > > + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", > > slot->tts_nvalid))); > > +} > > > > Similar to the above comment, the field name 'tts_nvalid' might also > > be changed in the future, let's use another name. > > Hmm. If the field name is changed, we need to change this > code. Yes, but if we use independe name in the NOTICE message we would not need to update the expected files. > > > --- > > +/* > > + * Export CopySendEndOfRow() for extensions. We want to keep > > + * CopySendEndOfRow() as a static function for > > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a > > + * compiler. > > + */ > > +void > > +CopyToStateFlush(CopyToState cstate) > > +{ > > + CopySendEndOfRow(cstate); > > +} > > > > Is there any reason to use a different name for public functions? > > In this patch set, I use "CopyFrom"/"CopyTo" prefixes for > public APIs for custom COPY FORMAT handler extensions. It > will help understanding related APIs. Is it strange in > PostgreSQL? I see your point. Probably we need to find a better name as the name CopyToStateFlush doesn't sound well like this API should be called only once at the end of a row (IOW user might try to call it multiple times to 'flush' the state while processing a row). How about CopyToEndOfRow()? > > > --- > > + /* For custom format implementation */ > > + void *opaque; /* private space */ > > > > How about renaming 'private'? > > We should not use "private" because it's a keyword in > C++. If we use "private" here, we can't include this file > from C++ code. Understood. > > > --- > > I've not reviewed the documentation patch yet but I think the patch > > seems to miss the updates to the description of the FORMAT option in > > the COPY command section. > > I defer this for now. We can revisit the last documentation > patch after we finalize our API. (Or could someone help us?) > > > I think we can reorganize the patch set as follows: > > > > 1. Create copyto_internal.h and change COPY_XXX to COPY_SOURCE_XXX and > > COPY_DEST_XXX accordingly. > > 2. Support custom format for both COPY TO and COPY FROM. > > 3. Expose necessary helper functions such as CopySendEndOfRow(). > > 4. Add CopyFromSkipErrorRow(). > > 5. Documentation. > > The attached v39 patch set uses the followings: > > 0001: Create copyto_internal.h and change COPY_XXX to > COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. > (Same as 1. in your suggestion) > 0002: Support custom format for both COPY TO and COPY FROM. > (Same as 2. in your suggestion) > 0003: Expose necessary helper functions such as CopySendEndOfRow() > and add CopyFromSkipErrorRow(). > (3. + 4. in your suggestion) > 0004: Define handler functions for built-in formats. > (Not included in your suggestion) > 0005: Documentation. (WIP) > (Same as 5. in your suggestion) Can we merge 0002 and 0004? > We can merge 0001 quickly, right? I don't think it makes sense to push only 0001 as it's a completely preliminary patch for subsequent patches. It would be prudent to push it once other patches are ready too. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-29 05:57 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-29 05:57 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Friday, March 28, 2025, Masahiko Sawada <[email protected]> wrote: > > > One problem in the following chunk I can see is: > > + qualified_format = stringToQualifiedNameList(format, NULL); > + DeconstructQualifiedName(qualified_format, &schema, &fmt); > + if (!schema || strcmp(schema, "pg_catalog") == 0) > + { > + if (strcmp(fmt, "csv") == 0) > + opts_out->csv_mode = true; > + else if (strcmp(fmt, "binary") == 0) > + opts_out->binary = true; > + } > > Non-qualified names depend on the search_path value so it's not > necessarily a built-in format. If the user specifies 'csv' with > seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily > sets csv_mode true. I think we can instead check if the retrieved > handler function's OID matches the built-in formats' ones. Also, it's > weired to me that cstate has csv_mode and binary fields even though > the format should have already been known by the callback functions. > I considered it a feature that a schema-less reference to text, csv, or binary always resolves to the core built-in handlers. As does an unspecified format default of text. To use an extension that chooses to override that format name would require schema qualification. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-29 08:57 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 2 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-03-29 08:57 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBKMNsO+b6wahb6847xwFci1JCfV+JykoMziVgiFxB6cw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 28 Mar 2025 22:37:03 -0700, Masahiko Sawada <[email protected]> wrote: >> I've added the following tests: >> >> * Wrong input type handler without namespace >> * Wrong input type handler with namespace >> * Wrong return type handler without namespace >> * Wrong return type handler with namespace >> * Wrong return value (Copy*Routine isn't returned) handler without namespace >> * Wrong return value (Copy*Routine isn't returned) handler with namespace >> * Nonexistent handler >> * Invalid qualified name >> * Valid handler without namespace and without search_path >> * Valid handler without namespace and with search_path >> * Valid handler with namespace > > Probably we can merge these newly added four files into one .sql file? I know that src/test/regress/sql/ uses this style (one .sql file includes many test patterns in one large category). I understand that the style is preferable in src/test/regress/sql/ because src/test/regress/sql/ has tests for many categories. But do we need to follow the style in src/test/modules/*/sql/ too? If we use the style in src/test/modules/*/sql/, we need to have only one .sql in src/test/modules/*/sql/ because src/test/modules/*/ are for each category. And the current .sql per sub-category style is easy to debug (at least for me). For example, if we try qualified name cases on debugger, we can use "\i sql/schema.sql" instead of extracting target statements from .sql that includes many unrelated statements. (Or we can use "\i sql/all.sql" and many GDB "continue"s.) BTW, it seems that src/test/modules/test_ddl_deparse/sql/ uses .sql per sub-category style. Should we use one .sql file for sql/test/modules/test_copy_format/sql/? If it's required for merging this patch set, I'll do it. > Also we need to add some comments to describe what these queries test. > For example, it's not clear to me at a glance what queries in > no-schema.sql are going to test as there is no comment there. Hmm. You refer no_schema.sql in 0002, right? ---- CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); \. COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); DROP TABLE public.test; DROP EXTENSION test_copy_format; ---- In general, COPY FORMAT tests focus on "COPY FROM WITH (FORMAT ...)" and "COPY TO WITH (FORMAT ...)". And the file name "no_schema" shows that it doesn't use qualified name. Based on this, I feel that the above content is very straightforward without any comment. What should we add as comments? For example, do we need the following comments? ---- -- This extension includes custom COPY handler: test_copy_format CREATE EXTENSION test_copy_format; -- Test data CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); -- Use custom COPY handler, test_copy_format, without -- schema for FROM. COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); \. -- Use custom COPY handler, test_copy_format, without -- schema for TO. COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); -- Cleanup DROP TABLE public.test; DROP EXTENSION test_copy_format; ---- > One problem in the following chunk I can see is: > > + qualified_format = stringToQualifiedNameList(format, NULL); > + DeconstructQualifiedName(qualified_format, &schema, &fmt); > + if (!schema || strcmp(schema, "pg_catalog") == 0) > + { > + if (strcmp(fmt, "csv") == 0) > + opts_out->csv_mode = true; > + else if (strcmp(fmt, "binary") == 0) > + opts_out->binary = true; > + } > > Non-qualified names depend on the search_path value so it's not > necessarily a built-in format. If the user specifies 'csv' with > seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily > sets csv_mode true. I think that we should always use built-in COPY handlers for (not-qualified) "text", "csv" and "binary" for compatibility. If we allow custom COPY handlers for (not-qualified) "text", "csv" and "binary", pg_dump or existing dump may be broken. Because we must use the same COPY handler when we dump (COPY TO) and we restore (COPY FROM). BTW, the current implementation always uses pg_catalog.{text,csv,binary} for (not-qualified) "text", "csv" and "binary" even when there are myschema.{text,csv,binary}. See src/test/modules/test_copy_format/sql/builtin.sql. But I haven't looked into it why... > I think we can instead check if the retrieved > handler function's OID matches the built-in formats' ones. I agree that the approach is clear than the current implementation. I'll use it when I create the next patch set. > Also, it's > weired to me that cstate has csv_mode and binary fields even though > the format should have already been known by the callback functions. You refer CopyFomratOptions::{csv_mode,binary} not Copy{To,From}StateData, right? And you suggest that we should replace all opts.csv_mode and opts.binary with opts.handler == F_CSV and opts.handler == F_BINARY, right? We can do it but I suggest that we do it as a refactoring (or cleanup) in a separated patch for easy to review. > Regarding the documentation for the existing options, it says "... > only when not using XXX format." some places, where XXX can be > replaced with binary or CSV. Once we support custom formats, 'non-CSV > mode' would actually include custom formats as well, so we need to > update the description too. I agree with you. >> > --- >> > +/* >> > + * Export CopySendEndOfRow() for extensions. We want to keep >> > + * CopySendEndOfRow() as a static function for >> > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a >> > + * compiler. >> > + */ >> > +void >> > +CopyToStateFlush(CopyToState cstate) >> > +{ >> > + CopySendEndOfRow(cstate); >> > +} >> > >> > Is there any reason to use a different name for public functions? >> >> In this patch set, I use "CopyFrom"/"CopyTo" prefixes for >> public APIs for custom COPY FORMAT handler extensions. It >> will help understanding related APIs. Is it strange in >> PostgreSQL? > > I see your point. Probably we need to find a better name as the name > CopyToStateFlush doesn't sound well like this API should be called > only once at the end of a row (IOW user might try to call it multiple > times to 'flush' the state while processing a row). How about > CopyToEndOfRow()? CopyToStateFlush() can be called multiple times in a row. It can also be called only once with multiple rows. Because it just flushes the current buffer. Existing CopySendEndOfRow() is called at the end of a row. (Buffer is flushed at the end of row.) So I think that the "EndOfRow" was chosen. Some custom COPY handlers may not be row based. For example, Apache Arrow COPY handler doesn't flush buffer for each row. So, we should provide "flush" API not "end of row" API for extensibility. >> The attached v39 patch set uses the followings: >> >> 0001: Create copyto_internal.h and change COPY_XXX to >> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. >> (Same as 1. in your suggestion) >> 0002: Support custom format for both COPY TO and COPY FROM. >> (Same as 2. in your suggestion) >> 0003: Expose necessary helper functions such as CopySendEndOfRow() >> and add CopyFromSkipErrorRow(). >> (3. + 4. in your suggestion) >> 0004: Define handler functions for built-in formats. >> (Not included in your suggestion) >> 0005: Documentation. (WIP) >> (Same as 5. in your suggestion) > > Can we merge 0002 and 0004? Can we do it when we merge this patch set if it's still desirable at the time? Because: * I think that separated 0002 and 0004 patches are easier to review than squashed 0002 and 0004 patch. * I still think that someone may don't like defining COPY handlers for built-in formats. If we don't define COPY handlers for built-in formats finally, we can just drop 0004. >> We can merge 0001 quickly, right? > > I don't think it makes sense to push only 0001 as it's a completely > preliminary patch for subsequent patches. It would be prudent to push > it once other patches are ready too. Hmm. I feel that 0001 is a refactoring category patch like merged patches. In general, distinct enum value names are easier to understand. BTW, does the "other patches" include the documentation patch...? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-29 16:12 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-29 16:12 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Sat, Mar 29, 2025 at 1:57 AM Sutou Kouhei <[email protected]> wrote: > * I still think that someone may don't like defining COPY > handlers for built-in formats. If we don't define COPY > handlers for built-in formats finally, we can just drop > 0004. > We should (and usually do) dog-food APIs when reasonable and this situation seems quite reasonable. I'd push back quite a bit about publishing this without any internal code leveraging it. > >> We can merge 0001 quickly, right? > > > > I don't think it makes sense to push only 0001 as it's a completely > > preliminary patch for subsequent patches. It would be prudent to push > > it once other patches are ready too. > > Hmm. I feel that 0001 is a refactoring category patch like > merged patches. In general, distinct enum value names are > easier to understand. > > I'm for pushing 0001. We've had copyfrom_internal.h for a while now and this seems like a simple refactor to make that area of the code cleaner via symmetry. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-29 16:48 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 3 siblings, 2 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-29 16:48 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote: > > The attached v39 patch set uses the followings: > > 0001: Create copyto_internal.h and change COPY_XXX to > COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. > (Same as 1. in your suggestion) > 0002: Support custom format for both COPY TO and COPY FROM. > (Same as 2. in your suggestion) > 0003: Expose necessary helper functions such as CopySendEndOfRow() > and add CopyFromSkipErrorRow(). > (3. + 4. in your suggestion) > 0004: Define handler functions for built-in formats. > (Not included in your suggestion) > 0005: Documentation. (WIP) > (Same as 5. in your suggestion) > > I don't think this module should be responsible for testing the validity of "qualified names in a string literal" behavior. Having some of the tests use a schema qualification, and I'd suggest explicit double-quoting/case-folding, wouldn't hurt just to demonstrate it's possible, and how extensions should be referenced, but definitely don't need tests to prove the broken cases are indeed broken. This relies on an existing API that has its own tests. It is definitely pointlessly redundant to have 6 tests that only differ from 6 other tests in their use of a schema qualification. I prefer keeping 0002 and 0004 separate. In particular, keeping the design choice of "unqualified internal format names ignore search_path" should stand out as its own commit. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-30 02:31 Sutou Kouhei <[email protected]> parent: David G. Johnston <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-03-30 02:31 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAKFQuwYF7VnYcS9dkfvdzt-dgftMB1DV0bjRcNC8-4iYGS+gjw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 29 Mar 2025 09:48:22 -0700, "David G. Johnston" <[email protected]> wrote: > I don't think this module should be responsible for testing the validity of > "qualified names in a string literal" behavior. Having some of the tests > use a schema qualification, and I'd suggest explicit > double-quoting/case-folding, wouldn't hurt just to demonstrate it's > possible, and how extensions should be referenced, but definitely don't > need tests to prove the broken cases are indeed broken. This relies on an > existing API that has its own tests. It is definitely pointlessly > redundant to have 6 tests that only differ from 6 other tests in their use > of a schema qualification. You suggest the followings, right? 1. Add tests for "Schema.Name" with mixed cases 2. Remove the following 6 tests in src/test/modules/test_copy_format/sql/invalid.sql ---- COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_input_type'); COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_input_type'); COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_return_type'); COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_return_type'); COPY public.test FROM stdin WITH (FORMAT 'test_schema.test_copy_format_wrong_return_value'); COPY public.test TO stdout WITH (FORMAT 'test_schema.test_copy_format_wrong_return_value'); ---- because we have the following 6 tests: ---- SET search_path = public,test_schema; COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_input_type'); COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_input_type'); COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_return_type'); COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_return_type'); COPY public.test FROM stdin WITH (FORMAT 'test_copy_format_wrong_return_value'); COPY public.test TO stdout WITH (FORMAT 'test_copy_format_wrong_return_value'); RESET search_path; ---- 3. Remove the following tests because the behavior must be tested in other places: ---- COPY public.test FROM stdin WITH (FORMAT 'nonexistent'); COPY public.test TO stdout WITH (FORMAT 'nonexistent'); COPY public.test FROM stdin WITH (FORMAT 'invalid.qualified.name'); COPY public.test TO stdout WITH (FORMAT 'invalid.qualified.name'); ---- Does it miss something? 1.: There is no difference between single-quoting and double-quoting here. Because the information what quote was used for the given FORMAT value isn't remained here. Should we update gram.y? 2.: I don't have a strong opinion for it. If nobody objects it, I'll remove them. 3.: I don't have a strong opinion for it. If nobody objects it, I'll remove them. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-31 17:05 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 3 siblings, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-31 17:05 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote: > > > --- > > +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { > > + .type = T_CopyFromRoutine, > > + .CopyFromInFunc = CopyFromInFunc, > > + .CopyFromStart = CopyFromStart, > > + .CopyFromOneRow = CopyFromOneRow, > > + .CopyFromEnd = CopyFromEnd, > > +}; > > > > In trying to document the current API I'm strongly disliking it. Namely, the amount of internal code an extension needs to care about/copy-paste to create a working handler. Presently, pg_type defines text and binary I/O routines and the text/csv formats use the text I/O while binary uses binary I/O - for all attributes. The CopyFromInFunc API allows for each attribute to somehow have its I/O format individualized. But I don't see how that is practical or useful, and it adds burden on API users. I suggest we remove both .CopyFromInFunc and .CopyFromStart/End and add a property to CopyFromRoutine (.ioMode?) with values of either Copy_IO_Text or Copy_IO_Binary and then just branch to either: CopyFromTextLikeInFunc & CopyFromTextLikeStart/End or CopyFromBinaryInFunc & CopyFromStart/End So, in effect, the only method an extension needs to write is converting to/from the 'serialized' form to the text/binary form (text being near unanimous). In a similar manner, the amount of boilerplate within CopyFromOneRow seems undesirable from an API perspective. cstate->cur_attname = NameStr(att->attname); cstate->cur_attval = string; if (string != NULL) nulls[m] = false; if (cstate->defaults[m]) { /* We must have switched into the per-tuple memory context */ Assert(econtext != NULL); Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); } /* * If ON_ERROR is specified with IGNORE, skip rows with soft errors */ else if (!InputFunctionCallSafe(&in_functions[m], string, typioparams[m], att->atttypmod, (Node *) cstate->escontext, &values[m])) { CopyFromSkipErrorRow(cstate); return true; } cstate->cur_attname = NULL; cstate->cur_attval = NULL; It seems to me that CopyFromOneRow could simply produce a *string collection, one cell per attribute, and NextCopyFrom could do all of the above on a for-loop over *string The pg_type and pg_proc catalogs are not extensible so the API can and should be limited to producing the, usually text, values that are ready to be passed into the text I/O routines and all the work to find and use types and functions left in the template code. I haven't looked at COPY TO but I am expecting much the same. The API should simply receive the content of the type I/O output routine (binary or text as it dictates) for each output attribute, by row, and be expected to take those values and produce a final output from them. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-31 18:51 Masahiko Sawada <[email protected]> parent: David G. Johnston <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-03-31 18:51 UTC (permalink / raw) To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Sat, Mar 29, 2025 at 9:49 AM David G. Johnston <[email protected]> wrote: > > On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote: >> >> >> The attached v39 patch set uses the followings: >> >> 0001: Create copyto_internal.h and change COPY_XXX to >> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. >> (Same as 1. in your suggestion) >> 0002: Support custom format for both COPY TO and COPY FROM. >> (Same as 2. in your suggestion) >> 0003: Expose necessary helper functions such as CopySendEndOfRow() >> and add CopyFromSkipErrorRow(). >> (3. + 4. in your suggestion) >> 0004: Define handler functions for built-in formats. >> (Not included in your suggestion) >> 0005: Documentation. (WIP) >> (Same as 5. in your suggestion) >> > > I prefer keeping 0002 and 0004 separate. In particular, keeping the design choice of "unqualified internal format names ignore search_path" should stand out as its own commit. What is the point of having separate commits for already-agreed design choices? I guess that it would make it easier to revert that decision. But I think it makes more sense that if we agree with "unqualified internal format names ignore search_path" the original commit includes that decision and describes it in the commit message. If we want to change that design based on the discussion later on, we can have a separate commit that makes that change and has the link to the discussion. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-31 19:35 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 3 replies; 127+ messages in thread From: Masahiko Sawada @ 2025-03-31 19:35 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Sat, Mar 29, 2025 at 1:57 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBKMNsO+b6wahb6847xwFci1JCfV+JykoMziVgiFxB6cw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 28 Mar 2025 22:37:03 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> I've added the following tests: > >> > >> * Wrong input type handler without namespace > >> * Wrong input type handler with namespace > >> * Wrong return type handler without namespace > >> * Wrong return type handler with namespace > >> * Wrong return value (Copy*Routine isn't returned) handler without namespace > >> * Wrong return value (Copy*Routine isn't returned) handler with namespace > >> * Nonexistent handler > >> * Invalid qualified name > >> * Valid handler without namespace and without search_path > >> * Valid handler without namespace and with search_path > >> * Valid handler with namespace > > > > Probably we can merge these newly added four files into one .sql file? > > I know that src/test/regress/sql/ uses this style (one .sql > file includes many test patterns in one large category). I > understand that the style is preferable in > src/test/regress/sql/ because src/test/regress/sql/ has > tests for many categories. > > But do we need to follow the style in > src/test/modules/*/sql/ too? If we use the style in > src/test/modules/*/sql/, we need to have only one .sql in > src/test/modules/*/sql/ because src/test/modules/*/ are for > each category. > > And the current .sql per sub-category style is easy to debug > (at least for me). For example, if we try qualified name > cases on debugger, we can use "\i sql/schema.sql" instead of > extracting target statements from .sql that includes many > unrelated statements. (Or we can use "\i sql/all.sql" and > many GDB "continue"s.) > > BTW, it seems that src/test/modules/test_ddl_deparse/sql/ > uses .sql per sub-category style. Should we use one .sql > file for sql/test/modules/test_copy_format/sql/? If it's > required for merging this patch set, I'll do it. I'm not sure that the regression test queries are categorized in the same way as in test_ddl_deparse. While the former have separate .sql files for different types of inputs (valid inputs and invalid inputs etc.) , which seems finer graind, the latter has .sql files for each DDL command. Most of the queries under test_copy_format/sql verifies the input patterns of the FORMAT option. I find that the regression tests included in that directory probably should focus on testing new callback APIs and some regression tests for FORMAT option handling can be moved into the normal regression test suite (e.g., in copy.sql or a new file like copy_format.sql). IIUC testing for invalid input patterns can be done even without creating artificial wrong handler functions. > > > Also we need to add some comments to describe what these queries test. > > For example, it's not clear to me at a glance what queries in > > no-schema.sql are going to test as there is no comment there. > > Hmm. You refer no_schema.sql in 0002, right? > > ---- > CREATE EXTENSION test_copy_format; > CREATE TABLE public.test (a smallint, b integer, c bigint); > INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); > COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); > \. > COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); > DROP TABLE public.test; > DROP EXTENSION test_copy_format; > ---- > > In general, COPY FORMAT tests focus on "COPY FROM WITH > (FORMAT ...)" and "COPY TO WITH (FORMAT ...)". And the file > name "no_schema" shows that it doesn't use qualified > name. Based on this, I feel that the above content is very > straightforward without any comment. > > What should we add as comments? For example, do we need the > following comments? > > ---- > -- This extension includes custom COPY handler: test_copy_format > CREATE EXTENSION test_copy_format; > -- Test data > CREATE TABLE public.test (a smallint, b integer, c bigint); > INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); > -- Use custom COPY handler, test_copy_format, without > -- schema for FROM. > COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); > \. > -- Use custom COPY handler, test_copy_format, without > -- schema for TO. > COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); > -- Cleanup > DROP TABLE public.test; > DROP EXTENSION test_copy_format; > ---- I'd like to see in the comment what the tests expect. Taking the following queries as an example, COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); \. COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); it would help readers understand the test case better if we have a comment like for example: -- Specify the custom format name without schema. We test if both -- COPY TO and COPY FROM can find the correct handler function -- in public schema. > > > One problem in the following chunk I can see is: > > > > + qualified_format = stringToQualifiedNameList(format, NULL); > > + DeconstructQualifiedName(qualified_format, &schema, &fmt); > > + if (!schema || strcmp(schema, "pg_catalog") == 0) > > + { > > + if (strcmp(fmt, "csv") == 0) > > + opts_out->csv_mode = true; > > + else if (strcmp(fmt, "binary") == 0) > > + opts_out->binary = true; > > + } > > > > Non-qualified names depend on the search_path value so it's not > > necessarily a built-in format. If the user specifies 'csv' with > > seach_patch = 'myschema, pg_catalog', the COPY command unnecessarily > > sets csv_mode true. > > I think that we should always use built-in COPY handlers for > (not-qualified) "text", "csv" and "binary" for > compatibility. If we allow custom COPY handlers for > (not-qualified) "text", "csv" and "binary", pg_dump or > existing dump may be broken. Because we must use the same > COPY handler when we dump (COPY TO) and we restore (COPY > FROM). I agreed. > > BTW, the current implementation always uses > pg_catalog.{text,csv,binary} for (not-qualified) "text", > "csv" and "binary" even when there are > myschema.{text,csv,binary}. See > src/test/modules/test_copy_format/sql/builtin.sql. But I > haven't looked into it why... Sorry, I don't follow that. IIUC test_copy_format extension doesn't create a handler function in myschema schema, and SQLs in builtin.sql seem to work as expected (specifying a non-qualified built-in format unconditionally uses the built-in format). > > > Also, it's > > weired to me that cstate has csv_mode and binary fields even though > > the format should have already been known by the callback functions. > > You refer CopyFomratOptions::{csv_mode,binary} not > Copy{To,From}StateData, right? Yes. I referred to the wrong one. > And you suggest that we > should replace all opts.csv_mode and opts.binary with > opts.handler == F_CSV and opts.handler == F_BINARY, right? > > We can do it but I suggest that we do it as a refactoring > (or cleanup) in a separated patch for easy to review. I think that csv_mode and binary are used mostly in ProcessCopyOptions() so probably we can use local variables for that. I find there are two other places where to use csv_mode: NextCopyFromRawFields() and CopyToTextLikeStart(). I think we can simply check the handler function's OID there, or we can define macros like COPY_FORMAT_IS_TEXT/CSV/BINARY checking the OID and use them there. > > >> > --- > >> > +/* > >> > + * Export CopySendEndOfRow() for extensions. We want to keep > >> > + * CopySendEndOfRow() as a static function for > >> > + * optimization. CopySendEndOfRow() calls in this file may be optimized by a > >> > + * compiler. > >> > + */ > >> > +void > >> > +CopyToStateFlush(CopyToState cstate) > >> > +{ > >> > + CopySendEndOfRow(cstate); > >> > +} > >> > > >> > Is there any reason to use a different name for public functions? > >> > >> In this patch set, I use "CopyFrom"/"CopyTo" prefixes for > >> public APIs for custom COPY FORMAT handler extensions. It > >> will help understanding related APIs. Is it strange in > >> PostgreSQL? > > > > I see your point. Probably we need to find a better name as the name > > CopyToStateFlush doesn't sound well like this API should be called > > only once at the end of a row (IOW user might try to call it multiple > > times to 'flush' the state while processing a row). How about > > CopyToEndOfRow()? > > CopyToStateFlush() can be called multiple times in a row. It > can also be called only once with multiple rows. Because it > just flushes the current buffer. > > Existing CopySendEndOfRow() is called at the end of a > row. (Buffer is flushed at the end of row.) So I think that > the "EndOfRow" was chosen. > > Some custom COPY handlers may not be row based. For example, > Apache Arrow COPY handler doesn't flush buffer for each row. > So, we should provide "flush" API not "end of row" API for > extensibility. Okay, understood. > > >> We can merge 0001 quickly, right? > > > > I don't think it makes sense to push only 0001 as it's a completely > > preliminary patch for subsequent patches. It would be prudent to push > > it once other patches are ready too. > > Hmm. I feel that 0001 is a refactoring category patch like > merged patches. In general, distinct enum value names are > easier to understand. Right, but the patches that have already been merged contributed to speed up COPY commands, but 0001 patch also introduces copyto_internal.h, which is not used by anyone in a case where the custom copy format patch is not merged. Without adding copyto_internal.h changing enum value names less makes sense to me. > BTW, does the "other patches" include the documentation > patch...? I think that when pushing the main custom COPY format patch, we need to include the documentation changes into it. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-03-31 20:42 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-03-31 20:42 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Mar 31, 2025 at 11:52 AM Masahiko Sawada <[email protected]> wrote: > On Sat, Mar 29, 2025 at 9:49 AM David G. Johnston > <[email protected]> wrote: > > > > On Wed, Mar 26, 2025 at 8:28 PM Sutou Kouhei <[email protected]> wrote: > >> > >> > >> The attached v39 patch set uses the followings: > >> > >> 0001: Create copyto_internal.h and change COPY_XXX to > >> COPY_SOURCE_XXX and COPY_DEST_XXX accordingly. > >> (Same as 1. in your suggestion) > >> 0002: Support custom format for both COPY TO and COPY FROM. > >> (Same as 2. in your suggestion) > >> 0003: Expose necessary helper functions such as CopySendEndOfRow() > >> and add CopyFromSkipErrorRow(). > >> (3. + 4. in your suggestion) > >> 0004: Define handler functions for built-in formats. > >> (Not included in your suggestion) > >> 0005: Documentation. (WIP) > >> (Same as 5. in your suggestion) > >> > > > > I prefer keeping 0002 and 0004 separate. In particular, keeping the > design choice of "unqualified internal format names ignore search_path" > should stand out as its own commit. > > What is the point of having separate commits for already-agreed design > choices? I guess that it would make it easier to revert that decision. > But I think it makes more sense that if we agree with "unqualified > internal format names ignore search_path" the original commit includes > that decision and describes it in the commit message. If we want to > change that design based on the discussion later on, we can have a > separate commit that makes that change and has the link to the > discussion. > Fair. Comment withdrawn. Though I was referring to the WIP patches; I figured the final patch would squash this all together in any case. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-04-06 16:20 David G. Johnston <[email protected]> parent: Sutou Kouhei <[email protected]> 3 siblings, 0 replies; 127+ messages in thread From: David G. Johnston @ 2025-04-06 16:20 UTC (permalink / raw) To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Sun, Apr 6, 2025 at 4:30 AM jian he <[email protected]> wrote: > > CREATE FUNCTION test_copy_format(internal) > RETURNS copy_handler > AS 'MODULE_PATHNAME', 'test_copy_format' > LANGUAGE C; > src/backend/commands/copy.c: ProcessCopyOptions > if (strcmp(fmt, "text") == 0) > /* default format */ ; > else if (strcmp(fmt, "csv") == 0) > opts_out->csv_mode = true; > else if (strcmp(fmt, "binary") == 0) > opts_out->binary = true; > else > { > List *qualified_format; > .... > } > what if our customized format name is one of "csv", "binary", "text", > then that ELSE branch will never be reached. > then our customized format is being shadowed? > > Yes. The user of your extension can specify a schema name to get access to your conflicting format name choice but all the existing code out there that relied on text/csv/binary being the built-in options continue to behave the same no matter the search_path. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-02 22:52 Masahiko Sawada <[email protected]> parent: Masahiko Sawada <[email protected]> 2 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-02 22:52 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, May 1, 2025 at 4:04 PM Michael Paquier <[email protected]> wrote: > > On Thu, May 01, 2025 at 12:15:30PM -0700, Masahiko Sawada wrote: > > In light of these concerns, I've been contemplating alternative > > interface designs. One promising approach would involve registering > > custom copy formats via a C function during module loading > > (specifically, in _PG_init()). This method would require extension > > authors to invoke a registration function, say > > RegisterCustomCopyFormat(), in _PG_init() as follows: > > > > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines", > > &JsonLinesCopyToRoutine, > > &JsonLinesCopyFromRoutine); > > > > The registration function would validate the format name and store it > > in TopMemoryContext. It would then return a unique identifier that can > > be used subsequently to reference the custom copy format extension. > > Hmm. How much should we care about the observability of the COPY > format used by a given backend? Storing this information in a > backend's TopMemoryContext is OK to get the extensibility basics to > work, but could it make sense to use some shmem state to allocate a > uint32 ID that could be shared by all backends. Contrary to EXPLAIN, > COPY commands usually run for a very long time, so I am wondering if > these APIs should be designed so as it would be possible to monitor > the format used. One layer where the format information could be made > available is the progress reporting view for COPY, for example. I can > also imagine a pgstats kind where we do COPY stats aggregates, with a > per-format pgstats kind, and sharing a fixed ID across multiple > backends is relevant (when flushing the stats at shutdown, we would > use a name/ID mapping like replication slots). > > I don't think that this needs to be relevant for the option part, just > for the format where, I suspect, we should store in a shmem array > based on the ID allocated the name of the format, the library of the > callback and the function name fed to load_external_function(). > > Note that custom LWLock and wait events use a shmem state for > monitoring purposes, where we are able to do ID->format name lookups > as much as format->ID lookups. Perhaps it's OK not to do that for > COPY, but I am wondering if we'd better design things from scratch > with states in shmem state knowing that COPY is a long-running > operation, and that if one mixes multiple formats they would most > likely want to know which formats are bottlenecks, through SQL. Cloud > providers would love that. Good point. It would make sense to have such information as a map on shmem. It might be better to use dshash here since a custom copy format module can be loaded at runtime. Or we can use dynahash with large enough elements. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 02:19 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 2 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-03 02:19 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBuEqcz2_+dpA3WTiDUF=FgudPBKwM+nvH+qHT-k4p5mA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 1 May 2025 12:15:30 -0700, Masahiko Sawada <[email protected]> wrote: > One of the primary considerations we need to address is the treatment > of the specified format name. The current patch set utilizes built-in > formats (namely 'csv', 'text', and 'binary') when the format name is > either unqualified or explicitly specified with 'pg_catalog' as the > schema. In all other cases, we search for custom format handler > functions based on the search_path. To be frank, I have reservations > about this interface design, as the dependence of the specified custom > format name on the search_path could potentially confuse users. How about requiring schema for all custom formats? Valid: COPY ... TO ... (FORMAT 'text'); COPY ... TO ... (FORMAT 'my_schema.jsonlines'); Invalid: COPY ... TO ... (FORMAT 'jsonlines'); -- no schema COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema If we require "schema" for all custom formats, we don't need to depend on search_path. > In light of these concerns, I've been contemplating alternative > interface designs. One promising approach would involve registering > custom copy formats via a C function during module loading > (specifically, in _PG_init()). This method would require extension > authors to invoke a registration function, say > RegisterCustomCopyFormat(), in _PG_init() as follows: > > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines", > &JsonLinesCopyToRoutine, > &JsonLinesCopyFromRoutine); > > The registration function would validate the format name and store it > in TopMemoryContext. It would then return a unique identifier that can > be used subsequently to reference the custom copy format extension. I don't object the suggested interface because I don't have a strong opinion how to implement this feature. Why do we need to assign a unique ID? For performance? For RegisterCustomCopyFormatOption()? I think that we don't need to use it so much in COPY. We don't need to use format name and assigned ID after we retrieve a corresponding Copy{To,From}Routine. Because all needed information are in Copy{To,From}Routine. > Extensions could register their own options within this > framework, for example: > > RegisterCustomCopyFormatOption(JsonLinesFormatId, > "custom_option", > custom_option_handler); Can we defer to discuss how to add support for custom options while we focus on the first implementation? Earlier patch sets with the current approach had custom options support but it's removed in the first implementation. (BTW, I think that it's not a good API because we want COPY FROM only options and COPY TO only options something like "compression level".) > This approach offers several advantages: it would eliminate the > search_path issue, provide greater flexibility, and potentially > simplify the overall interface for users and developers alike. What contributes to the "flexibility"? Developers can call multiple Register* functions in _PG_Init(), right? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 02:24 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-05-03 02:24 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoB82+MoP_RJ=zzhO9KaHK4LbfGjORkre34C7g-xsCdegQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 15:52:49 -0700, Masahiko Sawada <[email protected]> wrote: >> Hmm. How much should we care about the observability of the COPY >> format used by a given backend? Storing this information in a >> backend's TopMemoryContext is OK to get the extensibility basics to >> work, but could it make sense to use some shmem state to allocate a >> uint32 ID that could be shared by all backends. Contrary to EXPLAIN, >> COPY commands usually run for a very long time, so I am wondering if >> these APIs should be designed so as it would be possible to monitor >> the format used. One layer where the format information could be made >> available is the progress reporting view for COPY, for example. I can >> also imagine a pgstats kind where we do COPY stats aggregates, with a >> per-format pgstats kind, and sharing a fixed ID across multiple >> backends is relevant (when flushing the stats at shutdown, we would >> use a name/ID mapping like replication slots). >> >> I don't think that this needs to be relevant for the option part, just >> for the format where, I suspect, we should store in a shmem array >> based on the ID allocated the name of the format, the library of the >> callback and the function name fed to load_external_function(). >> >> Note that custom LWLock and wait events use a shmem state for >> monitoring purposes, where we are able to do ID->format name lookups >> as much as format->ID lookups. Perhaps it's OK not to do that for >> COPY, but I am wondering if we'd better design things from scratch >> with states in shmem state knowing that COPY is a long-running >> operation, and that if one mixes multiple formats they would most >> likely want to know which formats are bottlenecks, through SQL. Cloud >> providers would love that. > > Good point. It would make sense to have such information as a map on > shmem. It might be better to use dshash here since a custom copy > format module can be loaded at runtime. Or we can use dynahash with > large enough elements. If we don't need to assign an ID for each format, can we avoid it? If we implement it, is this approach more complex than the current table sampling method like approach? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 04:38 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-03 04:38 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, May 2, 2025 at 7:20 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBuEqcz2_+dpA3WTiDUF=FgudPBKwM+nvH+qHT-k4p5mA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 1 May 2025 12:15:30 -0700, > Masahiko Sawada <[email protected]> wrote: > > > One of the primary considerations we need to address is the treatment > > of the specified format name. The current patch set utilizes built-in > > formats (namely 'csv', 'text', and 'binary') when the format name is > > either unqualified or explicitly specified with 'pg_catalog' as the > > schema. In all other cases, we search for custom format handler > > functions based on the search_path. To be frank, I have reservations > > about this interface design, as the dependence of the specified custom > > format name on the search_path could potentially confuse users. > > How about requiring schema for all custom formats? > > Valid: > > COPY ... TO ... (FORMAT 'text'); > COPY ... TO ... (FORMAT 'my_schema.jsonlines'); > > Invalid: > > COPY ... TO ... (FORMAT 'jsonlines'); -- no schema > COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema > > If we require "schema" for all custom formats, we don't need > to depend on search_path. I'm concerned that users cannot use the same format name in the FORMAT option depending on which schema the handler function is created. > > > In light of these concerns, I've been contemplating alternative > > interface designs. One promising approach would involve registering > > custom copy formats via a C function during module loading > > (specifically, in _PG_init()). This method would require extension > > authors to invoke a registration function, say > > RegisterCustomCopyFormat(), in _PG_init() as follows: > > > > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines", > > &JsonLinesCopyToRoutine, > > &JsonLinesCopyFromRoutine); > > > > The registration function would validate the format name and store it > > in TopMemoryContext. It would then return a unique identifier that can > > be used subsequently to reference the custom copy format extension. > > I don't object the suggested interface because I don't have > a strong opinion how to implement this feature. > > Why do we need to assign a unique ID? For performance? For > RegisterCustomCopyFormatOption()? I think it's required for monitoring purposes for example. For instance, we can set the format ID in the progress information and the progress view can fetch the format name by the ID so that users can see what format is being used in the COPY command. > > > Extensions could register their own options within this > > framework, for example: > > > > RegisterCustomCopyFormatOption(JsonLinesFormatId, > > "custom_option", > > custom_option_handler); > > Can we defer to discuss how to add support for custom > options while we focus on the first implementation? Earlier > patch sets with the current approach had custom options > support but it's removed in the first implementation. I think we can skip the custom option patch for the first implementation but still need to discuss how we will be able to implement it to understand the big picture of this feature. Otherwise we could end up going the wrong direction. > > (BTW, I think that it's not a good API because we want COPY > FROM only options and COPY TO only options something like > "compression level".) Why does this matter in terms of API? I think that even with this API we can pass is_from to the option handler function so that it validates the option based on it. > > > This approach offers several advantages: it would eliminate the > > search_path issue, provide greater flexibility, and potentially > > simplify the overall interface for users and developers alike. > > What contributes to the "flexibility"? Developers can call > multiple Register* functions in _PG_Init(), right? I think that with a tablesample-like approach we need to do everything based on one handler function and callbacks returned from it whereas there is no such limitation with C API style. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 04:56 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-03 04:56 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBGRFStdVbHUcxL0QB8wn92J3Sn-6x=RhsSMuhepRH0NQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 21:38:32 -0700, Masahiko Sawada <[email protected]> wrote: >> How about requiring schema for all custom formats? >> >> Valid: >> >> COPY ... TO ... (FORMAT 'text'); >> COPY ... TO ... (FORMAT 'my_schema.jsonlines'); >> >> Invalid: >> >> COPY ... TO ... (FORMAT 'jsonlines'); -- no schema >> COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema >> >> If we require "schema" for all custom formats, we don't need >> to depend on search_path. > > I'm concerned that users cannot use the same format name in the FORMAT > option depending on which schema the handler function is created. I'm not sure that it's a problem or not. If users want to use the same format name, they can install the handler function to the same schema. >> Why do we need to assign a unique ID? For performance? For >> RegisterCustomCopyFormatOption()? > > I think it's required for monitoring purposes for example. For > instance, we can set the format ID in the progress information and the > progress view can fetch the format name by the ID so that users can > see what format is being used in the COPY command. How about setting the format name instead of the format ID in the progress information? > I think we can skip the custom option patch for the first > implementation but still need to discuss how we will be able to > implement it to understand the big picture of this feature. Otherwise > we could end up going the wrong direction. I think that we don't need to discuss it deeply because we have many options with this approach. We can call C functions in _PG_Init(). I think that this feature will not be a blocker of this approach. >> (BTW, I think that it's not a good API because we want COPY >> FROM only options and COPY TO only options something like >> "compression level".) > > Why does this matter in terms of API? I think that even with this API > we can pass is_from to the option handler function so that it > validates the option based on it. If we choose the API, each custom format developer needs to handle the case in handler function. For example, if we pass information whether this option is only for TO to PostgreSQL, ProcessCopyOptions() not handler functions can handle it. Anyway, I think that we don't need to discuss this deeply for now. >> What contributes to the "flexibility"? Developers can call >> multiple Register* functions in _PG_Init(), right? > > I think that with a tablesample-like approach we need to do everything > based on one handler function and callbacks returned from it whereas > there is no such limitation with C API style. Thanks for clarifying it. It seems that my understanding is correct. I hope that the flexibility is needed flexibility and too much flexibility doesn't introduce too much complexity. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 05:36 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 2 siblings, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-05-03 05:36 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Thursday, May 1, 2025, Masahiko Sawada <[email protected]> wrote: > > In light of these concerns, I've been contemplating alternative > interface designs. One promising approach would involve registering > custom copy formats via a C function during module loading > (specifically, in _PG_init()). This method would require extension > authors to invoke a registration function, say > RegisterCustomCopyFormat(), in _PG_init() as follows: > > JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines", > &JsonLinesCopyToRoutine, > &JsonLinesCopyFromRoutine); > > The registration function would validate the format name and store it > in TopMemoryContext. It would then return a unique identifier that can > be used subsequently to reference the custom copy format extension. > How does this fix the search_path concern? Are query writers supposed to put JsonLinesFormatId into their queries? Or are you just prohibiting a DBA from ever installing an extension that wants to register a format name that is already registered so that no namespace is ever required? ISTM accommodating a namespace for formats is required just like we do for virtually every other named object in the system. At least, if we want to play nice with extension authors. It doesn’t have to be within the existing pg_proc scope, we can create a new scope if desired, but abolishing it seems unwise. It would be more consistent with established policy if we didn’t make exceptions for text/csv/binary - if the DBA permits a text format to exist in a different schema and that schema appears first in the search_path, unqualified references to text would resolve to the non-core handler. We already protect ourselves with safe search_paths. This is really no different than if someone wanted to implement a now() function and people are putting pg_catalog from of existing usage. It’s the DBAs problem, not ours. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 05:57 Masahiko Sawada <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-03 05:57 UTC (permalink / raw) To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Fri, May 2, 2025 at 10:36 PM David G. Johnston <[email protected]> wrote: > > On Thursday, May 1, 2025, Masahiko Sawada <[email protected]> wrote: >> >> >> In light of these concerns, I've been contemplating alternative >> interface designs. One promising approach would involve registering >> custom copy formats via a C function during module loading >> (specifically, in _PG_init()). This method would require extension >> authors to invoke a registration function, say >> RegisterCustomCopyFormat(), in _PG_init() as follows: >> >> JsonLinesFormatId = RegisterCustomCopyFormat("jsonlines", >> &JsonLinesCopyToRoutine, >> &JsonLinesCopyFromRoutine); >> >> The registration function would validate the format name and store it >> in TopMemoryContext. It would then return a unique identifier that can >> be used subsequently to reference the custom copy format extension. > > > How does this fix the search_path concern? Are query writers supposed to put JsonLinesFormatId into their queries? Or are you just prohibiting a DBA from ever installing an extension that wants to register a format name that is already registered so that no namespace is ever required? Users can specify "jsonlines", passed in the first argument to the register function, to the COPY FORMAT option in this case. While JsonLinesFormatId is reserved for internal operations such as module processing and monitoring, any attempt to load another custom COPY format module named 'jsonlines' will result in an error. > ISTM accommodating a namespace for formats is required just like we do for virtually every other named object in the system. At least, if we want to play nice with extension authors. It doesn’t have to be within the existing pg_proc scope, we can create a new scope if desired, but abolishing it seems unwise. > > It would be more consistent with established policy if we didn’t make exceptions for text/csv/binary - if the DBA permits a text format to exist in a different schema and that schema appears first in the search_path, unqualified references to text would resolve to the non-core handler. We already protect ourselves with safe search_paths. This is really no different than if someone wanted to implement a now() function and people are putting pg_catalog from of existing usage. It’s the DBAs problem, not ours. I'm concerned about allowing multiple 'text' format implementations with identical names within the database, as this could lead to considerable confusion. When users specify 'text', it would be more logical to guarantee that the built-in 'text' format is consistently used. This principle aligns with other customizable components, such as custom resource managers, wait events, lightweight locks, and custom scans. These components maintain their built-in data/types and explicitly prevent the registration of duplicate names. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 06:02 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-03 06:02 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, May 2, 2025 at 9:56 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBGRFStdVbHUcxL0QB8wn92J3Sn-6x=RhsSMuhepRH0NQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 21:38:32 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> How about requiring schema for all custom formats? > >> > >> Valid: > >> > >> COPY ... TO ... (FORMAT 'text'); > >> COPY ... TO ... (FORMAT 'my_schema.jsonlines'); > >> > >> Invalid: > >> > >> COPY ... TO ... (FORMAT 'jsonlines'); -- no schema > >> COPY ... TO ... (FORMAT 'pg_catalog.text'); -- needless schema > >> > >> If we require "schema" for all custom formats, we don't need > >> to depend on search_path. > > > > I'm concerned that users cannot use the same format name in the FORMAT > > option depending on which schema the handler function is created. > > I'm not sure that it's a problem or not. If users want to > use the same format name, they can install the handler > function to the same schema. > > >> Why do we need to assign a unique ID? For performance? For > >> RegisterCustomCopyFormatOption()? > > > > I think it's required for monitoring purposes for example. For > > instance, we can set the format ID in the progress information and the > > progress view can fetch the format name by the ID so that users can > > see what format is being used in the COPY command. > > How about setting the format name instead of the format ID > in the progress information? The progress view can know only numbers. We need to extend the progress view infrastructure so that we can pass other data types. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 06:20 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-03 06:20 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoDnY2fhC7tp7jpn24AuwkeW-0YjFEtZbEfPwg8YcH6bAw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:02:25 -0700, Masahiko Sawada <[email protected]> wrote: > The progress view can know only numbers. We need to extend the > progress view infrastructure so that we can pass other data types. Sorry. Could you tell me what APIs referred here? pgstat_progress_*() functions in src/include/utils/backend_progress.h? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 06:37 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-05-03 06:37 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Friday, May 2, 2025, Masahiko Sawada <[email protected]> wrote: > > I'm concerned about allowing multiple 'text' format implementations > with identical names within the database, as this could lead to > considerable confusion. When users specify 'text', it would be more > logical to guarantee that the built-in 'text' format is consistently > used. Do you want to only give text/csv/binary this special treatment or also any future format name we ever decide to implement in core. If an extension takes up “xml” and we try to do that in core do we fail an upgrade because of the conflict, and make it impossible to actually use said extension? This principle aligns with other customizable components, such > as custom resource managers, wait events, lightweight locks, and > custom scans. These components maintain their built-in data/types and > explicitly prevent the registration of duplicate names. > I am totally lost on how any of those resemble this feature. I’m all for registration to enable additional options and features - but am against moving away from turning format into a namespaced identifier. This is a query-facing feature where namespaces are common and fundamentally required. I have some sympathy for the fact that until now one could not prefix text/binary/csv with pg_catalog to be fully safe, but in reality DBAs/query authors either put pg_catalog first in their search_path or make an informed decision when they deviate. That is the established precedent relevant to this feature. The power, and responsibility for education, lies with the user. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 06:37 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-03 06:37 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, May 2, 2025 at 11:20 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoDnY2fhC7tp7jpn24AuwkeW-0YjFEtZbEfPwg8YcH6bAw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:02:25 -0700, > Masahiko Sawada <[email protected]> wrote: > > > The progress view can know only numbers. We need to extend the > > progress view infrastructure so that we can pass other data types. > > Sorry. Could you tell me what APIs referred here? > pgstat_progress_*() functions in > src/include/utils/backend_progress.h? The progress information is stored in PgBackendStatus defined in backend_status.h: /* * Command progress reporting. Any command which wishes can advertise * that it is running by setting st_progress_command, * st_progress_command_target, and st_progress_param[]. * st_progress_command_target should be the OID of the relation which the * command targets (we assume there's just one, as this is meant for * utility commands), but the meaning of each element in the * st_progress_param array is command-specific. */ ProgressCommandType st_progress_command; Oid st_progress_command_target; int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM]; Then the progress view maps the numbers to the corresponding strings: CREATE VIEW pg_stat_progress_copy AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, S.relid AS relid, CASE S.param5 WHEN 1 THEN 'COPY FROM' WHEN 2 THEN 'COPY TO' END AS command, CASE S.param6 WHEN 1 THEN 'FILE' WHEN 2 THEN 'PROGRAM' WHEN 3 THEN 'PIPE' WHEN 4 THEN 'CALLBACK' END AS "type", S.param1 AS bytes_processed, S.param2 AS bytes_total, S.param3 AS tuples_processed, S.param4 AS tuples_excluded, S.param7 AS tuples_skipped FROM pg_stat_get_progress_info('COPY') AS S LEFT JOIN pg_database D ON S.datid = D.oid; So the idea is that the backend process sets the format ID somewhere in st_progress_param, and then the progress view calls a SQL function, say pg_stat_get_copy_format_name(), with the format ID that returns the corresponding format name. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 07:54 Masahiko Sawada <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-03 07:54 UTC (permalink / raw) To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Fri, May 2, 2025 at 11:37 PM David G. Johnston <[email protected]> wrote: > > On Friday, May 2, 2025, Masahiko Sawada <[email protected]> wrote: >> >> >> I'm concerned about allowing multiple 'text' format implementations >> with identical names within the database, as this could lead to >> considerable confusion. When users specify 'text', it would be more >> logical to guarantee that the built-in 'text' format is consistently >> used. > > > Do you want to only give text/csv/binary this special treatment or also any future format name we ever decide to implement in core. If an extension takes up “xml” and we try to do that in core do we fail an upgrade because of the conflict, and make it impossible to actually use said extension? I guess that's an extension author's responsibility to upgrade its extension so as to work with the new PostgreSQL version, or carefully choose the format name. They can even name '[extension_name].[format_name]' as a format name. Even with the current patch design (i.e., search_path affects handler function lookups), users would end up using the built-in 'xml' format without notice after upgrade, no? I guess that could introduce another problem. I think that we need to ensure that if users specify text/csv/binary the built-in formats are always used, to keep backward compatibility. > >> This principle aligns with other customizable components, such >> as custom resource managers, wait events, lightweight locks, and >> custom scans. These components maintain their built-in data/types and >> explicitly prevent the registration of duplicate names. > > > I am totally lost on how any of those resemble this feature. > > I’m all for registration to enable additional options and features - but am against moving away from turning format into a namespaced identifier. This is a query-facing feature where namespaces are common and fundamentally required. That's a fair concern. But isn't the format name ultimately just an option value, but not like a database object? As I mentioned above, I think we need to keep backward compatibility but treating the built-in formats special seems inconsistent with common name resolution behavior. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-03 14:42 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-05-03 14:42 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote: > I think that we need to ensure that if users specify text/csv/binary > the built-in formats are always used, to keep backward compatibility. That was my original thinking, but it’s inconsistent with how functions behave today. We don’t promise that installing extensions won’t cause existing code to change. > > > > I’m all for registration to enable additional options and features - but > am against moving away from turning format into a namespaced identifier. > This is a query-facing feature where namespaces are common and > fundamentally required. > > That's a fair concern. But isn't the format name ultimately just an > option value, but not like a database object? We get to decide that. And deciding in favor of “extensible database object in a namespace’ makes more sense - leveraging all that pre-existing design to play more nicely with extensions and give DBAs control. The SQL command to add one is “create function” instead of “create copy format”. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-04 05:02 Masahiko Sawada <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-04 05:02 UTC (permalink / raw) To: David G. Johnston <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Sat, May 3, 2025 at 7:42 AM David G. Johnston <[email protected]> wrote: > > On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote: > >> >> I think that we need to ensure that if users specify text/csv/binary >> the built-in formats are always used, to keep backward compatibility. > > > That was my original thinking, but it’s inconsistent with how functions behave today. We don’t promise that installing extensions won’t cause existing code to change. I'm skeptical about whether that's an acceptable backward compatibility breakage. >> > I’m all for registration to enable additional options and features - but am against moving away from turning format into a namespaced identifier. This is a query-facing feature where namespaces are common and fundamentally required. >> >> That's a fair concern. But isn't the format name ultimately just an >> option value, but not like a database object? > > > We get to decide that. And deciding in favor of “extensible database object in a namespace’ makes more sense - leveraging all that pre-existing design to play more nicely with extensions and give DBAs control. The SQL command to add one is “create function” instead of “create copy format”. I still don't fully understand why the FORMAT value alone needs to be treated like a schema-qualified object. If the concern is about name conflict with future built-in formats, I would argue that the same concern applies to custom EXPLAIN options and logical decoding plugins. To me, the benefit of treating the COPY FORMAT value as a schema-qualified object seems limited. Meanwhile, the risk of not protecting built-in formats like 'text', 'csv', and 'binary' is significant. If those names can be shadowed by extension via search_patch, we lose backward compatibility. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-04 05:27 David G. Johnston <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: David G. Johnston @ 2025-05-04 05:27 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote: > On Sat, May 3, 2025 at 7:42 AM David G. Johnston > <[email protected]> wrote: > > > > On Saturday, May 3, 2025, Masahiko Sawada <[email protected]> wrote: > > > >> > >> I think that we need to ensure that if users specify text/csv/binary > >> the built-in formats are always used, to keep backward compatibility. > > > > > > That was my original thinking, but it’s inconsistent with how functions > behave today. We don’t promise that installing extensions won’t cause > existing code to change. > > I'm skeptical about whether that's an acceptable backward > compatibility breakage. I’m skeptical you are correctly defining what backward-compatibility requires. Well, the only potential breakage is that we are searching for a matching function by signature without first limiting the mandated return type. But that is solve-able should anyone else see the problem as well. The global format name has its merits but neither it nor the namespaced format option suffer from breaking compatibility or policy. > > I still don't fully understand why the FORMAT value alone needs to be > treated like a schema-qualified object. If the concern is about name > conflict with future built-in formats, I would argue that the same > concern applies to custom EXPLAIN options and logical decoding > plugins. > Then maybe we have the same “problem” in those places. > > To me, the benefit of treating the COPY FORMAT value as a > schema-qualified object seems limited. Meanwhile, the risk of not > protecting built-in formats like 'text', 'csv', and 'binary' is > significant. Really? You think lots of extensions are going to choose to use these values even if they are permitted? Or are you concerned about attack surfaces? > If those names can be shadowed by extension via > search_patch, we lose backward compatibility. > This is not a definition of backward-compatibility that I am familiar with. If anything the ability for a DBA to arrange for such shadowing would be a feature enhancement. They can drop-in a more efficient or desirable implementation without having to change query code. In any case, I’m doubtful either of us can make a convincing enough argument to sway the other fully. Both options are plausible, IMO. Others need to chime in. David J. ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-09 08:51 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-09 08:51 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoD9CBjh4u6jdiE0tG-jvejw-GJN8fUPoQSVhKh36HW2NQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:37:46 -0700, Masahiko Sawada <[email protected]> wrote: > The progress information is stored in PgBackendStatus defined in > backend_status.h: > > /* > * Command progress reporting. Any command which wishes can advertise > * that it is running by setting st_progress_command, > * st_progress_command_target, and st_progress_param[]. > * st_progress_command_target should be the OID of the relation which the > * command targets (we assume there's just one, as this is meant for > * utility commands), but the meaning of each element in the > * st_progress_param array is command-specific. > */ > ProgressCommandType st_progress_command; > Oid st_progress_command_target; > int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM]; > > Then the progress view maps the numbers to the corresponding strings: > > CREATE VIEW pg_stat_progress_copy AS > SELECT > S.pid AS pid, S.datid AS datid, D.datname AS datname, > S.relid AS relid, > CASE S.param5 WHEN 1 THEN 'COPY FROM' > WHEN 2 THEN 'COPY TO' > END AS command, > CASE S.param6 WHEN 1 THEN 'FILE' > WHEN 2 THEN 'PROGRAM' > WHEN 3 THEN 'PIPE' > WHEN 4 THEN 'CALLBACK' > END AS "type", > S.param1 AS bytes_processed, > S.param2 AS bytes_total, > S.param3 AS tuples_processed, > S.param4 AS tuples_excluded, > S.param7 AS tuples_skipped > FROM pg_stat_get_progress_info('COPY') AS S > LEFT JOIN pg_database D ON S.datid = D.oid; Thanks. I didn't know about how to implement pg_stat_progress_copy. > So the idea is that the backend process sets the format ID somewhere > in st_progress_param, and then the progress view calls a SQL function, > say pg_stat_get_copy_format_name(), with the format ID that returns > the corresponding format name. Does it work when we use session_preload_libraries or the LOAD command? If we have 2 sessions and both of them load "jsonlines" COPY FORMAT extensions, what will be happened? For example: 1. Session 1: Register "jsonlines" 2. Session 2: Register "jsonlines" (Should global format ID <-> format name mapping be updated?) 3. Session 2: Close this session. Unregister "jsonlines". (Can we unregister COPY FORMAT extension?) (Should global format ID <-> format name mapping be updated?) 4. Session 1: Close this session. Unregister "jsonlines". (Can we unregister COPY FORMAT extension?) (Should global format ID <-> format name mapping be updated?) Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-09 09:41 Sutou Kouhei <[email protected]> parent: David G. Johnston <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-09 09:41 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAKFQuwaRDXANaL+QcT6LZRAem4rwkSwv9v+viv_mcR+Rex3quA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 3 May 2025 22:27:36 -0700, "David G. Johnston" <[email protected]> wrote: > In any case, I’m doubtful either of us can make a convincing enough > argument to sway the other fully. Both options are plausible, IMO. Others > need to chime in. I may misunderstand but here is the current summary, right? Proposed approaches to register custom COPY formats: a. Create a function that has the same name of custom COPY format b. Call a register function from _PG_init() FYI: I proposed c. approach that uses a. but it always requires schema name for format name in other e-mail. Users can register the same format name: a. Yes * Users can distinct the same format name by schema name * If format name doesn't have schema name, the used format depends on search_path * Pros: * Using schema for it is consistent with other PostgreSQL mechanisms * Custom format never conflict with built-in format. For example, an extension register "xml" and PostgreSQL adds "xml" later, they are never conflicted because PostgreSQL's "xml" is registered to pg_catalog. * Cons: Different format may be used with the same input. For example, "jsonlines" may choose "jsonlines" implemented by extension X or implemented by extension Y when search_path is different. b. No * Users can use "${schema}.${name}" for format name that mimics PostgreSQL's builtin schema (but it's just a string) Built-in formats (text/csv/binary) should be able to overwritten by extensions: a. (The current patch is no but David's answer is) Yes * Pros: Users can use drop-in replacement faster implementation without changing input * Cons: Users may overwrite them accidentally. It may break pg_dump result. (This is called as "backward incompatibility.") b. No Are there any missing or wrong items? If we can summarize the current discussion here correctly, others will be able to chime in this discussion. (At least I can do it.) Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-10 00:57 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-10 00:57 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, May 9, 2025 at 2:41 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAKFQuwaRDXANaL+QcT6LZRAem4rwkSwv9v+viv_mcR+Rex3quA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 3 May 2025 22:27:36 -0700, > "David G. Johnston" <[email protected]> wrote: > > > In any case, I’m doubtful either of us can make a convincing enough > > argument to sway the other fully. Both options are plausible, IMO. Others > > need to chime in. > > I may misunderstand but here is the current summary, right? Thank you for summarizing the discussion. > > Proposed approaches to register custom COPY formats: > a. Create a function that has the same name of custom COPY > format > b. Call a register function from _PG_init() > > FYI: I proposed c. approach that uses a. but it always > requires schema name for format name in other e-mail. With approach (c), do you mean that we require users to change all FORMAT option values like from 'text' to 'pg_catalog.text' after the upgrade? Or are we exempt the built-in formats? > > Users can register the same format name: > a. Yes > * Users can distinct the same format name by schema name > * If format name doesn't have schema name, the used > format depends on search_path > * Pros: > * Using schema for it is consistent with other > PostgreSQL mechanisms > * Custom format never conflict with built-in > format. For example, an extension register "xml" and > PostgreSQL adds "xml" later, they are never > conflicted because PostgreSQL's "xml" is registered > to pg_catalog. > * Cons: Different format may be used with the same > input. For example, "jsonlines" may choose > "jsonlines" implemented by extension X or implemented > by extension Y when search_path is different. > b. No > * Users can use "${schema}.${name}" for format name > that mimics PostgreSQL's builtin schema (but it's just > a string) > > > Built-in formats (text/csv/binary) should be able to > overwritten by extensions: > a. (The current patch is no but David's answer is) Yes > * Pros: Users can use drop-in replacement faster > implementation without changing input > * Cons: Users may overwrite them accidentally. > It may break pg_dump result. > (This is called as "backward incompatibility.") > b. No The summary matches my understanding. I think the second point is important. If we go with a tablesample-like API, I agree with David's point that all FORMAT values including the built-in formats should depend on the search_path value. While it provides a similar user experience to other database objects, there is a possibility that a COPY with built-in format could work differently on v19 than v18 or earlier depending on the search_path value. > Are there any missing or wrong items? I think the approach (b) provides more flexibility than (a) in terms of API design as with (a) we need to do everything based on one handler function and callbacks. > If we can summarize > the current discussion here correctly, others will be able > to chime in this discussion. (At least I can do it.) +1 Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-10 04:29 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-05-10 04:29 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, May 9, 2025 at 1:51 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoD9CBjh4u6jdiE0tG-jvejw-GJN8fUPoQSVhKh36HW2NQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 2 May 2025 23:37:46 -0700, > Masahiko Sawada <[email protected]> wrote: > > > The progress information is stored in PgBackendStatus defined in > > backend_status.h: > > > > /* > > * Command progress reporting. Any command which wishes can advertise > > * that it is running by setting st_progress_command, > > * st_progress_command_target, and st_progress_param[]. > > * st_progress_command_target should be the OID of the relation which the > > * command targets (we assume there's just one, as this is meant for > > * utility commands), but the meaning of each element in the > > * st_progress_param array is command-specific. > > */ > > ProgressCommandType st_progress_command; > > Oid st_progress_command_target; > > int64 st_progress_param[PGSTAT_NUM_PROGRESS_PARAM]; > > > > Then the progress view maps the numbers to the corresponding strings: > > > > CREATE VIEW pg_stat_progress_copy AS > > SELECT > > S.pid AS pid, S.datid AS datid, D.datname AS datname, > > S.relid AS relid, > > CASE S.param5 WHEN 1 THEN 'COPY FROM' > > WHEN 2 THEN 'COPY TO' > > END AS command, > > CASE S.param6 WHEN 1 THEN 'FILE' > > WHEN 2 THEN 'PROGRAM' > > WHEN 3 THEN 'PIPE' > > WHEN 4 THEN 'CALLBACK' > > END AS "type", > > S.param1 AS bytes_processed, > > S.param2 AS bytes_total, > > S.param3 AS tuples_processed, > > S.param4 AS tuples_excluded, > > S.param7 AS tuples_skipped > > FROM pg_stat_get_progress_info('COPY') AS S > > LEFT JOIN pg_database D ON S.datid = D.oid; > > Thanks. I didn't know about how to implement > pg_stat_progress_copy. > > > So the idea is that the backend process sets the format ID somewhere > > in st_progress_param, and then the progress view calls a SQL function, > > say pg_stat_get_copy_format_name(), with the format ID that returns > > the corresponding format name. > > Does it work when we use session_preload_libraries or the > LOAD command? If we have 2 sessions and both of them load > "jsonlines" COPY FORMAT extensions, what will be happened? > > For example: > > 1. Session 1: Register "jsonlines" > 2. Session 2: Register "jsonlines" > (Should global format ID <-> format name mapping > be updated?) > 3. Session 2: Close this session. > Unregister "jsonlines". > (Can we unregister COPY FORMAT extension?) > (Should global format ID <-> format name mapping > be updated?) > 4. Session 1: Close this session. > Unregister "jsonlines". > (Can we unregister COPY FORMAT extension?) > (Should global format ID <-> format name mapping > be updated?) I imagine that only for progress reporting purposes, I think session 1 and 2 can have different format IDs for the same 'jsonlines' if they load it by LOAD command. They can advertise the format IDs on the shmem and we can also provide a SQL function for the progress view that can get the format name by the format ID. Considering the possibility that we might want to use the format ID also in the cumulative statistics, we might want to strictly provide the unique format ID for each custom format as the format IDs are serialized to the pgstat file. One possible way to implement it is that we manage the custom format IDs in a wiki page like we do for custom cumulative statistics and custom RMGR[1][2]. That is, a custom format extension registers the format name along with the format ID that is pre-registered in the wiki page or the format ID (e.g. 128) indicating under development. If either the format name or format ID conflict with an already registered custom format extension, the registration function raises an error. And we preallocate enough format IDs for built-in formats. As for unregistration, I think that even if we provide an unregisteration API, it ultimately depends on whether or not custom format extensions call it in _PG_fini(). Regards, [1] https://wiki.postgresql.org/wiki/CustomCumulativeStats [2] https://wiki.postgresql.org/wiki/CustomWALResourceManagers Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-26 01:04 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-05-26 01:04 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBrSTmPyDai_QVR-XOe7PL722Dazm70A+FpvGy2hfSV9g@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 9 May 2025 17:57:35 -0700, Masahiko Sawada <[email protected]> wrote: >> Proposed approaches to register custom COPY formats: >> a. Create a function that has the same name of custom COPY >> format >> b. Call a register function from _PG_init() >> >> FYI: I proposed c. approach that uses a. but it always >> requires schema name for format name in other e-mail. > > With approach (c), do you mean that we require users to change all > FORMAT option values like from 'text' to 'pg_catalog.text' after the > upgrade? Or are we exempt the built-in formats? The latter. 'text' must be accepted because existing pg_dump results use 'text'. If we reject 'text', it's a big incompatibility. (We can't dump on old PostgreSQL and restore to new PostgreSQL.) >> Users can register the same format name: >> a. Yes >> * Users can distinct the same format name by schema name >> * If format name doesn't have schema name, the used >> format depends on search_path >> * Pros: >> * Using schema for it is consistent with other >> PostgreSQL mechanisms >> * Custom format never conflict with built-in >> format. For example, an extension register "xml" and >> PostgreSQL adds "xml" later, they are never >> conflicted because PostgreSQL's "xml" is registered >> to pg_catalog. >> * Cons: Different format may be used with the same >> input. For example, "jsonlines" may choose >> "jsonlines" implemented by extension X or implemented >> by extension Y when search_path is different. >> b. No >> * Users can use "${schema}.${name}" for format name >> that mimics PostgreSQL's builtin schema (but it's just >> a string) >> >> >> Built-in formats (text/csv/binary) should be able to >> overwritten by extensions: >> a. (The current patch is no but David's answer is) Yes >> * Pros: Users can use drop-in replacement faster >> implementation without changing input >> * Cons: Users may overwrite them accidentally. >> It may break pg_dump result. >> (This is called as "backward incompatibility.") >> b. No > > The summary matches my understanding. I think the second point is > important. If we go with a tablesample-like API, I agree with David's > point that all FORMAT values including the built-in formats should > depend on the search_path value. While it provides a similar user > experience to other database objects, there is a possibility that a > COPY with built-in format could work differently on v19 than v18 or > earlier depending on the search_path value. Thanks for sharing additional points. David said that the additional point case is a responsibility or DBA not PostgreSQL, right? As I already said, I don't have a strong opinion on which approach is better. My opinion for the (important) second point is no. I feel that the pros of a. isn't realistic. If users want to improve text/csv/binary performance (or something), they should improve PostgreSQL itself instead of replacing it as an extension. (Or they should create another custom copy format such as "faster_text" not "text".) So I'm OK with the approach b. >> Are there any missing or wrong items? > > I think the approach (b) provides more flexibility than (a) in terms > of API design as with (a) we need to do everything based on one > handler function and callbacks. Thanks for sharing this missing point. I have a concern that the flexibility may introduce needless complexity. If it's not a real concern, I'm OK with the approach b. >> If we can summarize >> the current discussion here correctly, others will be able >> to chime in this discussion. (At least I can do it.) > > +1 Are there any more people who are interested in custom COPY FORMAT implementation design? If no more people, let's decide it by us. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-05-26 01:27 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-05-26 01:27 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoAY_h-9nuhs14e3cyO_A2rH7==zuq+NPHkn9ggwyaXnPQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 9 May 2025 21:29:23 -0700, Masahiko Sawada <[email protected]> wrote: >> > So the idea is that the backend process sets the format ID somewhere >> > in st_progress_param, and then the progress view calls a SQL function, >> > say pg_stat_get_copy_format_name(), with the format ID that returns >> > the corresponding format name. >> >> Does it work when we use session_preload_libraries or the >> LOAD command? If we have 2 sessions and both of them load >> "jsonlines" COPY FORMAT extensions, what will be happened? >> >> For example: >> >> 1. Session 1: Register "jsonlines" >> 2. Session 2: Register "jsonlines" >> (Should global format ID <-> format name mapping >> be updated?) >> 3. Session 2: Close this session. >> Unregister "jsonlines". >> (Can we unregister COPY FORMAT extension?) >> (Should global format ID <-> format name mapping >> be updated?) >> 4. Session 1: Close this session. >> Unregister "jsonlines". >> (Can we unregister COPY FORMAT extension?) >> (Should global format ID <-> format name mapping >> be updated?) > > I imagine that only for progress reporting purposes, I think session 1 > and 2 can have different format IDs for the same 'jsonlines' if they > load it by LOAD command. They can advertise the format IDs on the > shmem and we can also provide a SQL function for the progress view > that can get the format name by the format ID. > > Considering the possibility that we might want to use the format ID > also in the cumulative statistics, we might want to strictly provide > the unique format ID for each custom format as the format IDs are > serialized to the pgstat file. One possible way to implement it is > that we manage the custom format IDs in a wiki page like we do for > custom cumulative statistics and custom RMGR[1][2]. That is, a custom > format extension registers the format name along with the format ID > that is pre-registered in the wiki page or the format ID (e.g. 128) > indicating under development. If either the format name or format ID > conflict with an already registered custom format extension, the > registration function raises an error. And we preallocate enough > format IDs for built-in formats. > > As for unregistration, I think that even if we provide an > unregisteration API, it ultimately depends on whether or not custom > format extensions call it in _PG_fini(). Thanks for sharing your idea. With the former ID issuing approach, it seems that we need a global format ID <-> name mapping and a per session registered format name list. The custom COPY FORMAT register function rejects the same format name, right? If we support both of shared_preload_libraries and session_preload_libraries/LOAD, we have different life time custom formats. It may introduce a complexity with the ID issuing approach. With the latter static ID approach, how to implement a function that converts format ID to format name? PostgreSQL itself doesn't know ID <-> name mapping in the Wiki page. It seems that custom COPY FORMAT implementation needs to register its name to PostgreSQL by itself. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-12 02:33 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2025-06-12 02:33 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, May 26, 2025 at 10:04:05AM +0900, Sutou Kouhei wrote: > As I already said, I don't have a strong opinion on which > approach is better. My opinion for the (important) second > point is no. I feel that the pros of a. isn't realistic. If > users want to improve text/csv/binary performance (or > something), they should improve PostgreSQL itself instead of > replacing it as an extension. (Or they should create another > custom copy format such as "faster_text" not "text".) Patches welcome. Andres may have a TODO board regarding that, I think. > So I'm OK with the approach b. Here is an opinion. Approach (b), that uses _PG_init() a function to register a custom format has the merit to be simple to implement and secure by "design", because it depends only on the fact that we can do a lookup based on the string defined in one or more DefElems. Adding a dependendy to search_path as you say could lead to surprising results. Using a shared ID when a COPY method is registered (like extension wait events) or an ID that's static in a backend (like EXPLAIN extensibility does) is an implementation difference that can be useful for monitoring, and only that AFAIK. If you want to implement method-based statistics for COPY, you will want to allocate one stats kind for each COPY method, because the stats stored will be aggregates of the COPY methods. The stats kind ID is something that should not be linked to the COPY method ID, because the stats kind ID is registered in its own dedicated path, and it would be hardcoded in the library where the COPY callbacks are defined. So you could have a stats kind with a fixed ID, and a COPY method ID that's linked to each backend like EXPLAIN does. One factor to take into account is how much freedom we are OK with giving to users when it comes to the deployment of custom COPY methods, and how popular these would be. Cloud is popular these days, so folks may want to be able to define pointers to functions that are run in something else than C, as long as the language is trusted. My take on this part is that we are not going to see many formats out there that would benefit from these callbacks, so asking for people to deploy a .so on disk that can only be LOAD'ed or registered with one of the preloading GUCs should be enough to satisfy most users, even if the barrier entry to get that only a cloud instead like RDS or Azure is higher. This has also the benefit in giving more control on the COPY internals to cloud providers, as they are the ones who would be in charge of saying if they're OK with a dedicated .so or not. Not the users themselves. We've had a lot of bad PR and false CVEs in the past with COPY FROM/TO PROGRAM and the fact that it requires superusers. Having something in this area that gives more freedom to the user with something like approach (a) (SQL functions allowed to define the callback) will, I suspect, bite us back hard. So, my opinion is to rely on _PG_init(), with a shared ID if you want to expose the method used somewhere for monitoring tools. You could as well implement the simpler set of APIs that allocates IDs local to each backend, like EXPLAIN, then consider later if shared IDs are really needed. The registration APIs don't have to be fixed in time across releases, they can be always improved in steps as required. What matters is ABI compatibility in the same major version once it is released. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-12 17:00 Masahiko Sawada <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-06-12 17:00 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jun 11, 2025 at 7:34 PM Michael Paquier <[email protected]> wrote: > > On Mon, May 26, 2025 at 10:04:05AM +0900, Sutou Kouhei wrote: > > As I already said, I don't have a strong opinion on which > > approach is better. My opinion for the (important) second > > point is no. I feel that the pros of a. isn't realistic. If > > users want to improve text/csv/binary performance (or > > something), they should improve PostgreSQL itself instead of > > replacing it as an extension. (Or they should create another > > custom copy format such as "faster_text" not "text".) > > Patches welcome. Andres may have a TODO board regarding that, I > think. > > > So I'm OK with the approach b. > > Here is an opinion. Thank you for the comments. > > Approach (b), that uses _PG_init() a function to register a custom > format has the merit to be simple to implement and secure by "design", > because it depends only on the fact that we can do a lookup based on > the string defined in one or more DefElems. Adding a dependendy to > search_path as you say could lead to surprising results. > > Using a shared ID when a COPY method is registered (like extension > wait events) or an ID that's static in a backend (like EXPLAIN > extensibility does) is an implementation difference that can be useful > for monitoring, and only that AFAIK. If you want to implement > method-based statistics for COPY, you will want to allocate one stats > kind for each COPY method, because the stats stored will be aggregates > of the COPY methods. The stats kind ID is something that should not > be linked to the COPY method ID, because the stats kind ID is > registered in its own dedicated path, and it would be hardcoded in the > library where the COPY callbacks are defined. So you could have a > stats kind with a fixed ID, and a COPY method ID that's linked to each > backend like EXPLAIN does. Good point. > > One factor to take into account is how much freedom we are OK with > giving to users when it comes to the deployment of custom COPY > methods, and how popular these would be. Cloud is popular these days, > so folks may want to be able to define pointers to functions that are > run in something else than C, as long as the language is trusted. My > take on this part is that we are not going to see many formats out > there that would benefit from these callbacks, so asking for people to > deploy a .so on disk that can only be LOAD'ed or registered with one > of the preloading GUCs should be enough to satisfy most users, even if > the barrier entry to get that only a cloud instead like RDS or Azure > is higher. This has also the benefit in giving more control on the > COPY internals to cloud providers, as they are the ones who would be > in charge of saying if they're OK with a dedicated .so or not. Not > the users themselves. We've had a lot of bad PR and false CVEs in the > past with COPY FROM/TO PROGRAM and the fact that it requires > superusers. Having something in this area that gives more freedom to > the user with something like approach (a) (SQL functions allowed to > define the callback) will, I suspect, bite us back hard. That's a valid point and I agree. > > So, my opinion is to rely on _PG_init(), with a shared ID if you want > to expose the method used somewhere for monitoring tools. You could > as well implement the simpler set of APIs that allocates IDs local to > each backend, like EXPLAIN, then consider later if shared IDs are > really needed. The registration APIs don't have to be fixed in time > across releases, they can be always improved in steps as required. > What matters is ABI compatibility in the same major version once it is > released. +1 to start with a simpler set of APIs. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-16 23:50 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-06-16 23:50 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBwxgfkMYxgPWyrLG-r8-ptVKjd=jhncY_QAaVJYhQQdw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 12 Jun 2025 10:00:12 -0700, Masahiko Sawada <[email protected]> wrote: >> So, my opinion is to rely on _PG_init(), with a shared ID if you want >> to expose the method used somewhere for monitoring tools. You could >> as well implement the simpler set of APIs that allocates IDs local to >> each backend, like EXPLAIN, then consider later if shared IDs are >> really needed. The registration APIs don't have to be fixed in time >> across releases, they can be always improved in steps as required. >> What matters is ABI compatibility in the same major version once it is >> released. > > +1 to start with a simpler set of APIs. OK. I'll implement the initial version with this design. (Allocating IDs local not shared.) Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-17 00:38 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Michael Paquier @ 2025-06-17 00:38 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote: > OK. I'll implement the initial version with this > design. (Allocating IDs local not shared.) Sounds good to me. Thanks Sutou-san! -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-18 03:59 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-06-18 03:59 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 17 Jun 2025 09:38:54 +0900, Michael Paquier <[email protected]> wrote: > On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote: >> OK. I'll implement the initial version with this >> design. (Allocating IDs local not shared.) > > Sounds good to me. Thanks Sutou-san! I've attached the v41 patch set that uses the C API approach with local (not shared) COPY routine management. 0001: This is same as 0001 in the v40 patch set. It just cleans up CopySource and CopyDest enums. 0002: This is the initial version of this approach. Here are some discussion points: 1. This provides 2 registration APIs (RegisterCopy{From,To}Routine(name, routine)) instead of 1 registration API (RegisterCopyFormat(name, from_routine, to_routine)). It's for simple implementation and easy to extend without breaking APIs in the future. (And some formats may provide only FROM routine or TO routine.) Is this design acceptable? FYI: RegisterCopy{From,To}Routine() uses the same logic as RegisterExtensionExplainOption(). 2. This allocates IDs internally but doesn't provide APIs that get them. Because it's not needed for now. We can provide GetExplainExtensionId() like API when we need it. Is this design acceptable? 3. I want to register the built-in COPY {FROM,TO} routines in the PostgreSQL initialization phase. Where should we do it? In 0002, it's done in InitPostgres() but I'm not sure whether it's a suitable location or not. 4. 0002 adds CopyFormatOptions::routine as union: @@ -87,9 +91,14 @@ typedef struct CopyFormatOptions CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ int64 reject_limit; /* maximum tolerable number of errors */ List *convert_select; /* list of column names (can be NIL) */ + union + { + const struct CopyFromRoutine *from; /* for COPY FROM */ + const struct CopyToRoutine *to; /* for COPY TO */ + } routine; /* routine to process the specified format */ } CopyFormatOptions; Because one of Copy{From,To}Routine is only needed at once. Is this union usage strange in PostgreSQL? 5. 0002 adds InitializeCopy{From,To}Routines() and GetCopy{From,To}Routine() that aren't used by COPY {FROM,TO} routine implementations to copyapi.h. Should we move them to other .h? If so, which .h should be used for them? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-24 02:59 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-06-24 02:59 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jun 18, 2025 at 12:59 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 17 Jun 2025 09:38:54 +0900, > Michael Paquier <[email protected]> wrote: > > > On Tue, Jun 17, 2025 at 08:50:37AM +0900, Sutou Kouhei wrote: > >> OK. I'll implement the initial version with this > >> design. (Allocating IDs local not shared.) > > > > Sounds good to me. Thanks Sutou-san! > > I've attached the v41 patch set that uses the C API approach > with local (not shared) COPY routine management. > > 0001: This is same as 0001 in the v40 patch set. It just > cleans up CopySource and CopyDest enums. > 0002: This is the initial version of this approach. Thank you for updating the patches! > Here are some discussion points: > > 1. This provides 2 registration APIs > (RegisterCopy{From,To}Routine(name, routine)) instead of > 1 registration API (RegisterCopyFormat(name, > from_routine, to_routine)). > > It's for simple implementation and easy to extend without > breaking APIs in the future. (And some formats may > provide only FROM routine or TO routine.) > > Is this design acceptable? With the single registration API idea, we can register the custom format routine that supports only FROM routine using the API like: RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL); Compared to this approach, what points do you think having separate two registration APIs is preferable in terms of extendability and API compatibility? I think it would be rather confusing for example if each COPY TO routine and COPY FROM routine is registered by different extensions with the same format name. > FYI: RegisterCopy{From,To}Routine() uses the same logic > as RegisterExtensionExplainOption(). I'm concerned that the patch has duplicated logics for the registration of COPY FROM and COPY TO. > > 2. This allocates IDs internally but doesn't provide APIs > that get them. Because it's not needed for now. > > We can provide GetExplainExtensionId() like API when we > need it. > > Is this design acceptable? +1 > > 3. I want to register the built-in COPY {FROM,TO} routines > in the PostgreSQL initialization phase. Where should we > do it? In 0002, it's done in InitPostgres() but I'm not > sure whether it's a suitable location or not. InitPostgres() is not a correct function as it's a process initialization function. Probably we don't necessarily need to register the built-in formats in the same way as custom formats. A simple solution would be to have separate arrays for built-in formats and custom formats and have the GetCopy[To|From]Routine() search both arrays (built-in array first). > 4. 0002 adds CopyFormatOptions::routine as union: > > @@ -87,9 +91,14 @@ typedef struct CopyFormatOptions > CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ > int64 reject_limit; /* maximum tolerable number of errors */ > List *convert_select; /* list of column names (can be NIL) */ > + union > + { > + const struct CopyFromRoutine *from; /* for COPY FROM */ > + const struct CopyToRoutine *to; /* for COPY TO */ > + } routine; /* routine to process the specified format */ > } CopyFormatOptions; > > Because one of Copy{From,To}Routine is only needed at > once. Is this union usage strange in PostgreSQL? I think we can live with having two fields as there are other options that are used only in either COPY FROM or COPY TO. > > 5. 0002 adds InitializeCopy{From,To}Routines() and > GetCopy{From,To}Routine() that aren't used by COPY > {FROM,TO} routine implementations to copyapi.h. Should we > move them to other .h? If so, which .h should be used for > them? As I commented at 3, I think it's better to avoid dynamically registering the built-in formats. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-24 05:11 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-06-24 05:11 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoA57owo6qYTPTxOtCjDmcuj1tGL1aGs95cvEnoLQvwF0A@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 11:59:17 +0900, Masahiko Sawada <[email protected]> wrote: >> 1. This provides 2 registration APIs >> (RegisterCopy{From,To}Routine(name, routine)) instead of >> 1 registration API (RegisterCopyFormat(name, >> from_routine, to_routine)). >> >> It's for simple implementation and easy to extend without >> breaking APIs in the future. (And some formats may >> provide only FROM routine or TO routine.) >> >> Is this design acceptable? > > With the single registration API idea, we can register the custom > format routine that supports only FROM routine using the API like: > > RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL); > > Compared to this approach, what points do you think having separate > two registration APIs is preferable in terms of extendability and API > compatibility? It's natural to add more related APIs with this approach. The single registration API provides one feature by one operation. If we use the RegisterCopyRoutine() for FROM and TO formats API, it's not natural that we add more related APIs. In this case, some APIs may provide multiple features by one operation and other APIs may provide single feature by one operation. Developers may be confused with the API. For example, developers may think "what does mean NULL here?" or "can we use NULL here?" for "RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL)". > I think it would be rather confusing for example if > each COPY TO routine and COPY FROM routine is registered by different > extensions with the same format name. Hmm, I don't think so. Who is confused by the case? DBA? Users who use COPY? Why is it confused? Do you assume the case that the same name is used for different format? For example, "json" is used for JSON Lines for COPY FROM and and normal JSON for COPY TO by different extensions. >> FYI: RegisterCopy{From,To}Routine() uses the same logic >> as RegisterExtensionExplainOption(). > > I'm concerned that the patch has duplicated logics for the > registration of COPY FROM and COPY TO. We can implement a convenient routine that can be used for RegisterExtensionExplainOption() and RegisterCopy{From,To}Routine() if it's needed. >> 3. I want to register the built-in COPY {FROM,TO} routines >> in the PostgreSQL initialization phase. Where should we >> do it? In 0002, it's done in InitPostgres() but I'm not >> sure whether it's a suitable location or not. > > InitPostgres() is not a correct function as it's a process > initialization function. Probably we don't necessarily need to > register the built-in formats in the same way as custom formats. A > simple solution would be to have separate arrays for built-in formats > and custom formats and have the GetCopy[To|From]Routine() search both > arrays (built-in array first). We had a discussion that we should dog-food APIs: https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4... > We should (and usually do) dog-food APIs when reasonable > and this situation seems quite reasonable. In this case, we don't need to dog-food APIs, right? >> 4. 0002 adds CopyFormatOptions::routine as union: >> >> @@ -87,9 +91,14 @@ typedef struct CopyFormatOptions >> CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ >> int64 reject_limit; /* maximum tolerable number of errors */ >> List *convert_select; /* list of column names (can be NIL) */ >> + union >> + { >> + const struct CopyFromRoutine *from; /* for COPY FROM */ >> + const struct CopyToRoutine *to; /* for COPY TO */ >> + } routine; /* routine to process the specified format */ >> } CopyFormatOptions; >> >> Because one of Copy{From,To}Routine is only needed at >> once. Is this union usage strange in PostgreSQL? > > I think we can live with having two fields as there are other options > that are used only in either COPY FROM or COPY TO. OK. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-24 06:24 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-06-24 06:24 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jun 24, 2025 at 2:11 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoA57owo6qYTPTxOtCjDmcuj1tGL1aGs95cvEnoLQvwF0A@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 11:59:17 +0900, > Masahiko Sawada <[email protected]> wrote: > > >> 1. This provides 2 registration APIs > >> (RegisterCopy{From,To}Routine(name, routine)) instead of > >> 1 registration API (RegisterCopyFormat(name, > >> from_routine, to_routine)). > >> > >> It's for simple implementation and easy to extend without > >> breaking APIs in the future. (And some formats may > >> provide only FROM routine or TO routine.) > >> > >> Is this design acceptable? > > > > With the single registration API idea, we can register the custom > > format routine that supports only FROM routine using the API like: > > > > RegisterCopyRoutine("new-format", NewFormatFromRoutine, NULL); > > > > Compared to this approach, what points do you think having separate > > two registration APIs is preferable in terms of extendability and API > > compatibility? > > It's natural to add more related APIs with this > approach. The single registration API provides one feature > by one operation. If we use the RegisterCopyRoutine() for > FROM and TO formats API, it's not natural that we add more > related APIs. In this case, some APIs may provide multiple > features by one operation and other APIs may provide single > feature by one operation. Developers may be confused with > the API. For example, developers may think "what does mean > NULL here?" or "can we use NULL here?" for > "RegisterCopyRoutine("new-format", NewFormatFromRoutine, > NULL)". We can document it in the comment for the registration function. > > > > I think it would be rather confusing for example if > > each COPY TO routine and COPY FROM routine is registered by different > > extensions with the same format name. > > Hmm, I don't think so. Who is confused by the case? DBA? > Users who use COPY? Why is it confused? > > Do you assume the case that the same name is used for > different format? For example, "json" is used for JSON Lines > for COPY FROM and and normal JSON for COPY TO by different > extensions. Suppose that extension-A implements only CopyToRoutine for the custom-format-X with the format name 'myformat' and extension-B implements only CopyFromRoutine for the custom-format-Y with the same name, if users load both extension-A and extension-B, it seems to me that extension-A registers the custom-format-X format as 'myformat' only with CopyToRoutine, and extension-B overwrites the 'myformat' registration by adding custom-format-Y's CopyFromRoutine. However, if users register extension-C that implements both routines with the format name 'myformat', they can register neither extension-A nor extension-B, which seems to me that we don't allow overwriting the registration in this case. I think the core issue appears to be the internal management of custom format entries but the current patch does enable registration overwriting in the former case (extension-A and extension-B case). > > >> FYI: RegisterCopy{From,To}Routine() uses the same logic > >> as RegisterExtensionExplainOption(). > > > > I'm concerned that the patch has duplicated logics for the > > registration of COPY FROM and COPY TO. > > We can implement a convenient routine that can be used for > RegisterExtensionExplainOption() and > RegisterCopy{From,To}Routine() if it's needed. I meant there are duplicated codes in COPY FROM and COPY TO. For instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have the same logic. > > >> 3. I want to register the built-in COPY {FROM,TO} routines > >> in the PostgreSQL initialization phase. Where should we > >> do it? In 0002, it's done in InitPostgres() but I'm not > >> sure whether it's a suitable location or not. > > > > InitPostgres() is not a correct function as it's a process > > initialization function. Probably we don't necessarily need to > > register the built-in formats in the same way as custom formats. A > > simple solution would be to have separate arrays for built-in formats > > and custom formats and have the GetCopy[To|From]Routine() search both > > arrays (built-in array first). > > We had a discussion that we should dog-food APIs: > > https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4... > > > We should (and usually do) dog-food APIs when reasonable > > and this situation seems quite reasonable. > > In this case, we don't need to dog-food APIs, right? Yes, I think so. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-24 07:10 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-06-24 07:10 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoC8-d=GF-hOvGqUyq2xFg=QGpYfCiWJbcp4wcn0UidrPw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 15:24:23 +0900, Masahiko Sawada <[email protected]> wrote: >> It's natural to add more related APIs with this >> approach. The single registration API provides one feature >> by one operation. If we use the RegisterCopyRoutine() for >> FROM and TO formats API, it's not natural that we add more >> related APIs. In this case, some APIs may provide multiple >> features by one operation and other APIs may provide single >> feature by one operation. Developers may be confused with >> the API. For example, developers may think "what does mean >> NULL here?" or "can we use NULL here?" for >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine, >> NULL)". > > We can document it in the comment for the registration function. I think that API that can be understandable without the additional note is better API than API that needs some notes. Why do you suggest the RegisterCopyRoutine("new-format", NewFormatFromRoutine, NewFormatToRoutine) API? You want to remove the duplicated codes in RegisterCopy{From,To}Routine(), right? I think that we can do it by creating a convenient function that has the duplicated codes extracted from RegisterCopy{From,To}Routine() and RegisterExtensionExplainOption(). BTW, what do you think about my answer (one feature by one operation API is more extendable API) for your question (extendability and API compatibility)? > Suppose that extension-A implements only CopyToRoutine for the > custom-format-X with the format name 'myformat' and extension-B > implements only CopyFromRoutine for the custom-format-Y with the same > name, if users load both extension-A and extension-B, it seems to me > that extension-A registers the custom-format-X format as 'myformat' > only with CopyToRoutine, and extension-B overwrites the 'myformat' > registration by adding custom-format-Y's CopyFromRoutine. However, if > users register extension-C that implements both routines with the > format name 'myformat', they can register neither extension-A nor > extension-B, which seems to me that we don't allow overwriting the > registration in this case. Do you assume that users use extension-A, extension-B and extension-C without reading their documentation? If users read their documentation before users use them, users can know all of them use the same format name 'myformat' and which extension provides Copy{From,To}Routine. In this case, these users (who don't read documentation) will be confused with the RegisterCopyRoutine("new-format", NewFormatFromRoutine, NewFormatToRoutine) API too. Do we really need to care about this case? > I think the core issue appears to be the internal management of custom > format entries but the current patch does enable registration > overwriting in the former case (extension-A and extension-B case). This is the same behavior as existing custom EXPLAIN option implementation. Should we use different behavior here? >> >> FYI: RegisterCopy{From,To}Routine() uses the same logic >> >> as RegisterExtensionExplainOption(). >> > >> > I'm concerned that the patch has duplicated logics for the >> > registration of COPY FROM and COPY TO. >> >> We can implement a convenient routine that can be used for >> RegisterExtensionExplainOption() and >> RegisterCopy{From,To}Routine() if it's needed. > > I meant there are duplicated codes in COPY FROM and COPY TO. For > instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have > the same logic. Yes, I understand it. I wanted to say that we can remove the duplicated codes by introducing a RegisterSomething() function that can be used by RegisterExtensionExplainOption() and RegisterCopy{From,To}Routine(): void RegisterSomething(...) { /* Common codes in RegisterExtensionExplainOption() and RegisterCopy{From,To}Routine() ... */ } void RegisterExtensionExplainOption(...) { RegisterSomething(...); } void RegisterCopyFromRoutine(...) { RegisterSomething(...); } void RegisterCopyToRoutine(...) { RegisterSomething(...); } You think that this approach can't remove the duplicated codes, right? >> > InitPostgres() is not a correct function as it's a process >> > initialization function. Probably we don't necessarily need to >> > register the built-in formats in the same way as custom formats. A >> > simple solution would be to have separate arrays for built-in formats >> > and custom formats and have the GetCopy[To|From]Routine() search both >> > arrays (built-in array first). >> >> We had a discussion that we should dog-food APIs: >> >> https://www.postgresql.org/message-id/flat/CAKFQuwaCHhrS%2BRE4p_OO6d7WEskd9b86-2cYcvChNkrP%2B7PJ7A%4... >> >> > We should (and usually do) dog-food APIs when reasonable >> > and this situation seems quite reasonable. >> >> In this case, we don't need to dog-food APIs, right? > > Yes, I think so. OK. I don't have a strong opinion for it. If nobody objects it, I'll do it when I update the patch set. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-24 15:48 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-06-24 15:48 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jun 24, 2025 at 4:10 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoC8-d=GF-hOvGqUyq2xFg=QGpYfCiWJbcp4wcn0UidrPw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 24 Jun 2025 15:24:23 +0900, > Masahiko Sawada <[email protected]> wrote: > > >> It's natural to add more related APIs with this > >> approach. The single registration API provides one feature > >> by one operation. If we use the RegisterCopyRoutine() for > >> FROM and TO formats API, it's not natural that we add more > >> related APIs. In this case, some APIs may provide multiple > >> features by one operation and other APIs may provide single > >> feature by one operation. Developers may be confused with > >> the API. For example, developers may think "what does mean > >> NULL here?" or "can we use NULL here?" for > >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine, > >> NULL)". > > > > We can document it in the comment for the registration function. > > I think that API that can be understandable without the > additional note is better API than API that needs some > notes. I don't see much difference in this case. > > Why do you suggest the RegisterCopyRoutine("new-format", > NewFormatFromRoutine, NewFormatToRoutine) API? You want to > remove the duplicated codes in > RegisterCopy{From,To}Routine(), right? No. I think that if extensions are likely to support both CopyToRoutine and CopyFromRoutine in most cases, it would be simpler to register the custom format using a single API. Registering CopyToRoutine and CopyFromRoutine separately seems redundant to me. > > BTW, what do you think about my answer (one feature by one > operation API is more extendable API) for your question > (extendability and API compatibility)? Could you provide some examples? It seems to me that even if we provide the single API for the registration we can provide other APIs differently. For example, if we want to provide an API to register a custom option, we can provide RegisterCopyToOption() and RegisterCopyFromOption(). > > > Suppose that extension-A implements only CopyToRoutine for the > > custom-format-X with the format name 'myformat' and extension-B > > implements only CopyFromRoutine for the custom-format-Y with the same > > name, if users load both extension-A and extension-B, it seems to me > > that extension-A registers the custom-format-X format as 'myformat' > > only with CopyToRoutine, and extension-B overwrites the 'myformat' > > registration by adding custom-format-Y's CopyFromRoutine. However, if > > users register extension-C that implements both routines with the > > format name 'myformat', they can register neither extension-A nor > > extension-B, which seems to me that we don't allow overwriting the > > registration in this case. > > Do you assume that users use extension-A, extension-B and > extension-C without reading their documentation? If users > read their documentation before users use them, users can > know all of them use the same format name 'myformat' and > which extension provides Copy{From,To}Routine. > > In this case, these users (who don't read documentation) > will be confused with the RegisterCopyRoutine("new-format", > NewFormatFromRoutine, NewFormatToRoutine) API too. Do we > really need to care about this case? My point is about the consistency of registration behavior. I think that we should raise an error if the custom format name that an extension tries to register already exists. Therefore I'm not sure why installing extension-A+B is okay but installing extension-C+A or extension-C+B is not okay? We can think that's an extension-A's choice not to implement CopyFromRoutine for the 'myformat' format so extension-B should not change it. > > > I think the core issue appears to be the internal management of custom > > format entries but the current patch does enable registration > > overwriting in the former case (extension-A and extension-B case). > > This is the same behavior as existing custom EXPLAIN option > implementation. Should we use different behavior here? I think that unlike custom EXPLAIN options, it's better to raise an error or a warning if the custom format name (or combination of format name and COPY direction) that an extension tries to register already exists. > >> >> FYI: RegisterCopy{From,To}Routine() uses the same logic > >> >> as RegisterExtensionExplainOption(). > >> > > >> > I'm concerned that the patch has duplicated logics for the > >> > registration of COPY FROM and COPY TO. > >> > >> We can implement a convenient routine that can be used for > >> RegisterExtensionExplainOption() and > >> RegisterCopy{From,To}Routine() if it's needed. > > > > I meant there are duplicated codes in COPY FROM and COPY TO. For > > instance, RegisterCopyFromRoutine() and RegisterCopyToRoutine() have > > the same logic. > > Yes, I understand it. I wanted to say that we can remove the > duplicated codes by introducing a RegisterSomething() > function that can be used by > RegisterExtensionExplainOption() and > RegisterCopy{From,To}Routine(): > > void > RegisterSomething(...) > { > /* Common codes in RegisterExtensionExplainOption() and > RegisterCopy{From,To}Routine() > ... > */ > } > > void > RegisterExtensionExplainOption(...) > { > RegisterSomething(...); > } > > void > RegisterCopyFromRoutine(...) > { > RegisterSomething(...); > } > > void > RegisterCopyToRoutine(...) > { > RegisterSomething(...); > } > > You think that this approach can't remove the duplicated > codes, right? Well, no, I just meant we don't need to do that. Custom EXPLAIN option and custom COPY format are different features and have different requirements. I think while we don't need to remove duplicates between them at least at this stage we need to remove the duplicate between COPY TO registration code and COPY TO's one. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-25 07:35 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-06-25 07:35 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900, Masahiko Sawada <[email protected]> wrote: >> >> It's natural to add more related APIs with this >> >> approach. The single registration API provides one feature >> >> by one operation. If we use the RegisterCopyRoutine() for >> >> FROM and TO formats API, it's not natural that we add more >> >> related APIs. In this case, some APIs may provide multiple >> >> features by one operation and other APIs may provide single >> >> feature by one operation. Developers may be confused with >> >> the API. For example, developers may think "what does mean >> >> NULL here?" or "can we use NULL here?" for >> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine, >> >> NULL)". >> > >> > We can document it in the comment for the registration function. >> >> I think that API that can be understandable without the >> additional note is better API than API that needs some >> notes. > > I don't see much difference in this case. OK. It seems that we can't agree on which API is better. I've implemented your idea as the v42 patch set. Can we proceed this proposal with this approach? What is the next step? > No. I think that if extensions are likely to support both > CopyToRoutine and CopyFromRoutine in most cases, it would be simpler > to register the custom format using a single API. Registering > CopyToRoutine and CopyFromRoutine separately seems redundant to me. I don't think so. In general, extensions are implemented step by step. Extension developers will not implement CopyToRoutine and CopyFromRoutine at once even if extensions implement both of CopyToRoutine and CopyFromRoutine eventually. > Could you provide some examples? It seems to me that even if we > provide the single API for the registration we can provide other APIs > differently. For example, if we want to provide an API to register a > custom option, we can provide RegisterCopyToOption() and > RegisterCopyFromOption(). Yes. We can mix different style APIs. In general, consistent style APIs is easier to use than mixed style APIs. If it's not an important point in PostgreSQL API design, my point is meaningless. (Sorry, I'm not familiar with PostgreSQL API design.) > My point is about the consistency of registration behavior. I think > that we should raise an error if the custom format name that an > extension tries to register already exists. Therefore I'm not sure why > installing extension-A+B is okay but installing extension-C+A or > extension-C+B is not okay? We can think that's an extension-A's choice > not to implement CopyFromRoutine for the 'myformat' format so > extension-B should not change it. I think that it's the users' responsibility. I think that it's more convenient that users can mix extension-A+B (A provides only TO format and B provides only FROM format) than users can't mix them. I think that extension-A doesn't want to prohibit FROM format in the case. Extension-A just doesn't care about FROM format. FYI: Both of extension-C+A and extension-C+B are OK when we update not raising an error existing format. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-06-30 06:00 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-06-30 06:00 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jun 25, 2025 at 4:35 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900, > Masahiko Sawada <[email protected]> wrote: > > >> >> It's natural to add more related APIs with this > >> >> approach. The single registration API provides one feature > >> >> by one operation. If we use the RegisterCopyRoutine() for > >> >> FROM and TO formats API, it's not natural that we add more > >> >> related APIs. In this case, some APIs may provide multiple > >> >> features by one operation and other APIs may provide single > >> >> feature by one operation. Developers may be confused with > >> >> the API. For example, developers may think "what does mean > >> >> NULL here?" or "can we use NULL here?" for > >> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine, > >> >> NULL)". > >> > > >> > We can document it in the comment for the registration function. > >> > >> I think that API that can be understandable without the > >> additional note is better API than API that needs some > >> notes. > > > > I don't see much difference in this case. > > OK. It seems that we can't agree on which API is better. > > I've implemented your idea as the v42 patch set. Can we > proceed this proposal with this approach? What is the next > step? I'll review the patches. In the meanwhile could you update the documentation accordingly? > > > No. I think that if extensions are likely to support both > > CopyToRoutine and CopyFromRoutine in most cases, it would be simpler > > to register the custom format using a single API. Registering > > CopyToRoutine and CopyFromRoutine separately seems redundant to me. > > I don't think so. In general, extensions are implemented > step by step. Extension developers will not implement > CopyToRoutine and CopyFromRoutine at once even if extensions > implement both of CopyToRoutine and CopyFromRoutine > eventually. Hmm, I think if the extension eventually implements both directions, it would make sense to provide the single API. > > > Could you provide some examples? It seems to me that even if we > > provide the single API for the registration we can provide other APIs > > differently. For example, if we want to provide an API to register a > > custom option, we can provide RegisterCopyToOption() and > > RegisterCopyFromOption(). > > Yes. We can mix different style APIs. In general, consistent > style APIs is easier to use than mixed style APIs. If it's > not an important point in PostgreSQL API design, my point is > meaningless. (Sorry, I'm not familiar with PostgreSQL API > design.) As far as I know, there is no standard for PostgreSQL API design, but I don't find any weirdness in this design. > > > My point is about the consistency of registration behavior. I think > > that we should raise an error if the custom format name that an > > extension tries to register already exists. Therefore I'm not sure why > > installing extension-A+B is okay but installing extension-C+A or > > extension-C+B is not okay? We can think that's an extension-A's choice > > not to implement CopyFromRoutine for the 'myformat' format so > > extension-B should not change it. > > I think that it's the users' responsibility. I think that > it's more convenient that users can mix extension-A+B (A > provides only TO format and B provides only FROM format) > than users can't mix them. I think that extension-A doesn't > want to prohibit FROM format in the case. Extension-A just > doesn't care about FROM format. > > FYI: Both of extension-C+A and extension-C+B are OK when we > update not raising an error existing format. I want to keep the basic design that one custom format comes from one extension because it's straightforward for both of us and users and easy to maintain format ID. IIUC we somewhat agreed on this design in the previous API design (TABLESAMPLE like API). Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-13 18:28 Masahiko Sawada <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Masahiko Sawada @ 2025-07-13 18:28 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jun 30, 2025 at 3:00 PM Masahiko Sawada <[email protected]> wrote: > > On Wed, Jun 25, 2025 at 4:35 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAD21AoC19fV5Ujs-1r24MNU+hwTQUeZMEnaJDjSFwHLMMdFi0Q@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 25 Jun 2025 00:48:46 +0900, > > Masahiko Sawada <[email protected]> wrote: > > > > >> >> It's natural to add more related APIs with this > > >> >> approach. The single registration API provides one feature > > >> >> by one operation. If we use the RegisterCopyRoutine() for > > >> >> FROM and TO formats API, it's not natural that we add more > > >> >> related APIs. In this case, some APIs may provide multiple > > >> >> features by one operation and other APIs may provide single > > >> >> feature by one operation. Developers may be confused with > > >> >> the API. For example, developers may think "what does mean > > >> >> NULL here?" or "can we use NULL here?" for > > >> >> "RegisterCopyRoutine("new-format", NewFormatFromRoutine, > > >> >> NULL)". > > >> > > > >> > We can document it in the comment for the registration function. > > >> > > >> I think that API that can be understandable without the > > >> additional note is better API than API that needs some > > >> notes. > > > > > > I don't see much difference in this case. > > > > OK. It seems that we can't agree on which API is better. > > > > I've implemented your idea as the v42 patch set. Can we > > proceed this proposal with this approach? What is the next > > step? > > I'll review the patches. I've reviewed the 0001 and 0002 patches. The API implemented in the 0002 patch looks good to me, but I'm concerned about the capsulation of copy state data. With the v42 patches, we pass the whole CopyToStateData to the extension codes, but most of the fields in CopyToStateData are internal working state data that shouldn't be exposed to extensions. I think we need to sort out which fields are exposed or not. That way, it would be safer and we would be able to avoid exposing copyto_internal.h and extensions would not need to include copyfrom_internal.h. I've implemented a draft patch for that idea. In the 0001 patch, I moved fields that are related to internal working state from CopyToStateData to CopyToExectuionData. COPY routine APIs pass a pointer of CopyToStateData but extensions can access only fields except for CopyToExectuionData. In the 0002 patch, I've implemented the registration API and some related APIs based on your v42 patch. I've made similar changes to COPY FROM codes too. The patch is a very PoC phase and we would need to scrutinize the fields that should or should not be exposed. Feedback is very welcome. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [application/octet-stream] 0001-Refactor-CopyToStateData-and-CopyFromStateData.patch (80.9K, ../../CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com/2-0001-Refactor-CopyToStateData-and-CopyFromStateData.patch) download | inline diff: From f80e7ea77a4faa2abd64782e9bdfc20ad8d0eff3 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Sun, 13 Jul 2025 06:50:40 -0700 Subject: [PATCH 1/2] Refactor CopyToStateData and CopyFromStateData. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- src/backend/commands/copyfrom.c | 238 ++++++------ src/backend/commands/copyfromparse.c | 446 ++++++++++++----------- src/backend/commands/copyto.c | 111 +++--- src/include/commands/copy.h | 55 ++- src/include/commands/copyfrom_internal.h | 22 +- 5 files changed, 457 insertions(+), 415 deletions(-) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index fbbbc09a97b..03779838654 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -168,6 +168,7 @@ CopyFromGetRoutine(const CopyFormatOptions *opts) static void CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc) { + CopyFromExecutionData *edata = cstate->edata; AttrNumber attr_count; /* @@ -175,24 +176,24 @@ CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc) * converted input data. Otherwise, we can just point input_buf to the * same buffer as raw_buf. */ - if (cstate->need_transcoding) + if (cstate->edata->need_transcoding) { - cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); - cstate->input_buf_index = cstate->input_buf_len = 0; + edata->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); + edata->input_buf_index = edata->input_buf_len = 0; } else - cstate->input_buf = cstate->raw_buf; - cstate->input_reached_eof = false; + edata->input_buf = edata->raw_buf; + edata->input_reached_eof = false; - initStringInfo(&cstate->line_buf); + initStringInfo(&edata->line_buf); /* * Create workspace for CopyReadAttributes results; used by CSV and text * format. */ attr_count = list_length(cstate->attnumlist); - cstate->max_fields = attr_count; - cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); + edata->max_fields = attr_count; + edata->raw_fields = (char **) palloc(attr_count * sizeof(char *)); } /* @@ -254,48 +255,49 @@ void CopyFromErrorCallback(void *arg) { CopyFromState cstate = (CopyFromState) arg; + CopyFromExecutionData *edata = cstate->edata; - if (cstate->relname_only) + if (edata->relname_only) { errcontext("COPY %s", - cstate->cur_relname); + edata->cur_relname); return; } if (cstate->opts.binary) { /* can't usefully display the data */ - if (cstate->cur_attname) + if (edata->cur_attname) errcontext("COPY %s, line %" PRIu64 ", column %s", - cstate->cur_relname, - cstate->cur_lineno, - cstate->cur_attname); + edata->cur_relname, + edata->cur_lineno, + edata->cur_attname); else errcontext("COPY %s, line %" PRIu64, - cstate->cur_relname, - cstate->cur_lineno); + edata->cur_relname, + edata->cur_lineno); } else { - if (cstate->cur_attname && cstate->cur_attval) + if (edata->cur_attname && edata->cur_attval) { /* error is relevant to a particular column */ char *attval; - attval = CopyLimitPrintoutLength(cstate->cur_attval); + attval = CopyLimitPrintoutLength(edata->cur_attval); errcontext("COPY %s, line %" PRIu64 ", column %s: \"%s\"", - cstate->cur_relname, - cstate->cur_lineno, - cstate->cur_attname, + edata->cur_relname, + edata->cur_lineno, + edata->cur_attname, attval); pfree(attval); } - else if (cstate->cur_attname) + else if (edata->cur_attname) { /* error is relevant to a particular column, value is NULL */ errcontext("COPY %s, line %" PRIu64 ", column %s: null input", - cstate->cur_relname, - cstate->cur_lineno, - cstate->cur_attname); + edata->cur_relname, + edata->cur_lineno, + edata->cur_attname); } else { @@ -304,21 +306,21 @@ CopyFromErrorCallback(void *arg) * * If line_buf still contains the correct line, print it. */ - if (cstate->line_buf_valid) + if (edata->line_buf_valid) { char *lineval; - lineval = CopyLimitPrintoutLength(cstate->line_buf.data); + lineval = CopyLimitPrintoutLength(edata->line_buf.data); errcontext("COPY %s, line %" PRIu64 ": \"%s\"", - cstate->cur_relname, - cstate->cur_lineno, lineval); + edata->cur_relname, + edata->cur_lineno, lineval); pfree(lineval); } else { errcontext("COPY %s, line %" PRIu64, - cstate->cur_relname, - cstate->cur_lineno); + edata->cur_relname, + edata->cur_lineno); } } } @@ -448,6 +450,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, int64 *processed) { CopyFromState cstate = miinfo->cstate; + CopyFromExecutionData *edata = cstate->edata; EState *estate = miinfo->estate; int nused = buffer->nused; ResultRelInfo *resultRelInfo = buffer->resultRelInfo; @@ -469,8 +472,8 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, * We suppress error context information other than the relation name, * if one of the operations below fails. */ - Assert(!cstate->relname_only); - cstate->relname_only = true; + Assert(!edata->relname_only); + edata->relname_only = true; while (sent < nused) { @@ -514,7 +517,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, ExecARInsertTriggers(estate, resultRelInfo, slot, NIL, - cstate->transition_capture); + edata->transition_capture); } } @@ -528,14 +531,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, ExecClearTuple(slots[i]); /* reset relname_only */ - cstate->relname_only = false; + edata->relname_only = false; } else { CommandId mycid = miinfo->mycid; int ti_options = miinfo->ti_options; - bool line_buf_valid = cstate->line_buf_valid; - uint64 save_cur_lineno = cstate->cur_lineno; + bool line_buf_valid = edata->line_buf_valid; + uint64 save_cur_lineno = edata->cur_lineno; MemoryContext oldcontext; Assert(buffer->bistate != NULL); @@ -544,7 +547,7 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, * Print error context information correctly, if one of the operations * below fails. */ - cstate->line_buf_valid = false; + edata->line_buf_valid = false; /* * table_multi_insert may leak memory, so switch to short-lived memory @@ -569,14 +572,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, { List *recheckIndexes; - cstate->cur_lineno = buffer->linenos[i]; + edata->cur_lineno = buffer->linenos[i]; recheckIndexes = ExecInsertIndexTuples(resultRelInfo, buffer->slots[i], estate, false, false, NULL, NIL, false); ExecARInsertTriggers(estate, resultRelInfo, slots[i], recheckIndexes, - cstate->transition_capture); + edata->transition_capture); list_free(recheckIndexes); } @@ -588,10 +591,10 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, (resultRelInfo->ri_TrigDesc->trig_insert_after_row || resultRelInfo->ri_TrigDesc->trig_insert_new_table)) { - cstate->cur_lineno = buffer->linenos[i]; + edata->cur_lineno = buffer->linenos[i]; ExecARInsertTriggers(estate, resultRelInfo, slots[i], NIL, - cstate->transition_capture); + edata->transition_capture); } ExecClearTuple(slots[i]); @@ -603,8 +606,8 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, *processed); /* reset cur_lineno and line_buf_valid to what they were */ - cstate->line_buf_valid = line_buf_valid; - cstate->cur_lineno = save_cur_lineno; + edata->line_buf_valid = line_buf_valid; + edata->cur_lineno = save_cur_lineno; } /* Mark that all slots are free */ @@ -778,6 +781,7 @@ CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, uint64 CopyFrom(CopyFromState cstate) { + CopyFromExecutionData *edata = cstate->edata; ResultRelInfo *resultRelInfo; ResultRelInfo *target_resultRelInfo; ResultRelInfo *prevResultRelInfo = NULL; @@ -801,10 +805,10 @@ CopyFrom(CopyFromState cstate) bool leafpart_use_multi_insert = false; Assert(cstate->rel); - Assert(list_length(cstate->range_table) == 1); + Assert(list_length(edata->range_table) == 1); if (cstate->opts.on_error != COPY_ON_ERROR_STOP) - Assert(cstate->escontext); + Assert(edata->escontext); /* * The target must be a plain, foreign, or partitioned relation, or have @@ -913,7 +917,7 @@ CopyFrom(CopyFromState cstate) * index-entry-making machinery. (There used to be a huge amount of code * here that basically duplicated execUtils.c ...) */ - ExecInitRangeTable(estate, cstate->range_table, cstate->rteperminfos, + ExecInitRangeTable(estate, edata->range_table, edata->rteperminfos, bms_make_singleton(1)); resultRelInfo = target_resultRelInfo = makeNode(ResultRelInfo); ExecInitResultRelation(estate, resultRelInfo, 1); @@ -968,7 +972,7 @@ CopyFrom(CopyFromState cstate) * transition capture is active, we also set it in mtstate, which is * passed to ExecFindPartition() below. */ - cstate->transition_capture = mtstate->mt_transition_capture = + edata->transition_capture = mtstate->mt_transition_capture = MakeTransitionCaptureState(cstate->rel->trigdesc, RelationGetRelid(cstate->rel), CMD_INSERT); @@ -980,9 +984,9 @@ CopyFrom(CopyFromState cstate) if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) proute = ExecSetupPartitionTupleRouting(estate, cstate->rel); - if (cstate->whereClause) - cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause), - &mtstate->ps); + if (edata->whereClause) + edata->qualexpr = ExecInitQual(castNode(List, edata->whereClause), + &mtstate->ps); /* * It's generally more efficient to prepare a bunch of tuples for @@ -1025,7 +1029,7 @@ CopyFrom(CopyFromState cstate) */ insertMethod = CIM_SINGLE; } - else if (cstate->volatile_defexprs) + else if (edata->volatile_defexprs) { /* * Can't support multi-inserts if there are any volatile default @@ -1038,7 +1042,7 @@ CopyFrom(CopyFromState cstate) */ insertMethod = CIM_SINGLE; } - else if (contain_volatile_functions(cstate->whereClause)) + else if (contain_volatile_functions(edata->whereClause)) { /* * Can't support multi-inserts if there are any volatile function @@ -1150,7 +1154,7 @@ CopyFrom(CopyFromState cstate) break; if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE && - cstate->escontext->error_occurred) + edata->escontext->error_occurred) { /* * Soft error occurred, skip this tuple and just make @@ -1158,14 +1162,14 @@ CopyFrom(CopyFromState cstate) * don't set details_wanted and error_data is not to be filled, * just resetting error_occurred is enough. */ - cstate->escontext->error_occurred = false; + edata->escontext->error_occurred = false; /* Report that this tuple was skipped by the ON_ERROR clause */ pgstat_progress_update_param(PROGRESS_COPY_TUPLES_SKIPPED, - cstate->num_errors); + edata->num_errors); if (cstate->opts.reject_limit > 0 && - cstate->num_errors > cstate->opts.reject_limit) + edata->num_errors > cstate->opts.reject_limit) ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("skipped more than REJECT_LIMIT (%" PRId64 ") rows due to data type incompatibility", @@ -1186,11 +1190,11 @@ CopyFrom(CopyFromState cstate) /* Triggers and stuff need to be invoked in query context. */ MemoryContextSwitchTo(oldcontext); - if (cstate->whereClause) + if (edata->whereClause) { econtext->ecxt_scantuple = myslot; /* Skip items that don't match COPY's WHERE clause */ - if (!ExecQual(cstate->qualexpr, econtext)) + if (!ExecQual(edata->qualexpr, econtext)) { /* * Report that this tuple was filtered out by the WHERE @@ -1266,8 +1270,8 @@ CopyFrom(CopyFromState cstate) * we can just remember the original unconverted tuple to avoid a * needless round trip conversion. */ - if (cstate->transition_capture != NULL) - cstate->transition_capture->tcs_original_insert_tuple = + if (edata->transition_capture != NULL) + edata->transition_capture->tcs_original_insert_tuple = !has_before_insert_row_trig ? myslot : NULL; /* @@ -1379,8 +1383,8 @@ CopyFrom(CopyFromState cstate) /* Add this tuple to the tuple buffer */ CopyMultiInsertInfoStore(&multiInsertInfo, resultRelInfo, myslot, - cstate->line_buf.len, - cstate->cur_lineno); + edata->line_buf.len, + edata->cur_lineno); /* * If enough inserts have queued up, then flush all @@ -1440,7 +1444,7 @@ CopyFrom(CopyFromState cstate) /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, myslot, - recheckIndexes, cstate->transition_capture); + recheckIndexes, edata->transition_capture); list_free(recheckIndexes); } @@ -1468,13 +1472,13 @@ CopyFrom(CopyFromState cstate) error_context_stack = errcallback.previous; if (cstate->opts.on_error != COPY_ON_ERROR_STOP && - cstate->num_errors > 0 && + edata->num_errors > 0 && cstate->opts.log_verbosity >= COPY_LOG_VERBOSITY_DEFAULT) ereport(NOTICE, errmsg_plural("%" PRIu64 " row was skipped due to data type incompatibility", "%" PRIu64 " rows were skipped due to data type incompatibility", - cstate->num_errors, - cstate->num_errors)); + edata->num_errors, + edata->num_errors)); if (bistate != NULL) FreeBulkInsertState(bistate); @@ -1482,7 +1486,7 @@ CopyFrom(CopyFromState cstate) MemoryContextSwitchTo(oldcontext); /* Execute AFTER STATEMENT insertion triggers */ - ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture); + ExecASInsertTriggers(estate, target_resultRelInfo, edata->transition_capture); /* Handle queued AFTER triggers */ AfterTriggerEndQuery(estate); @@ -1536,6 +1540,7 @@ BeginCopyFrom(ParseState *pstate, List *options) { CopyFromState cstate; + CopyFromExecutionData *edata; bool pipe = (filename == NULL); TupleDesc tupDesc; AttrNumber num_phys_attrs, @@ -1560,15 +1565,18 @@ BeginCopyFrom(ParseState *pstate, /* Allocate workspace and zero all fields */ cstate = (CopyFromStateData *) palloc0(sizeof(CopyFromStateData)); + edata = (CopyFromExecutionData *) palloc0(sizeof(CopyFromExecutionData)); + cstate->edata = edata; + /* * We allocate everything used by a cstate in a new memory context. This * avoids memory leaks during repeated use of COPY in a query. */ - cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext, - "COPY", - ALLOCSET_DEFAULT_SIZES); + edata->copycontext = AllocSetContextCreate(CurrentMemoryContext, + "COPY", + ALLOCSET_DEFAULT_SIZES); - oldcontext = MemoryContextSwitchTo(cstate->copycontext); + oldcontext = MemoryContextSwitchTo(edata->copycontext); /* Extract options from the statement node tree */ ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options); @@ -1617,19 +1625,19 @@ BeginCopyFrom(ParseState *pstate, /* Set up soft error handler for ON_ERROR */ if (cstate->opts.on_error != COPY_ON_ERROR_STOP) { - cstate->escontext = makeNode(ErrorSaveContext); - cstate->escontext->type = T_ErrorSaveContext; - cstate->escontext->error_occurred = false; + edata->escontext = makeNode(ErrorSaveContext); + edata->escontext->type = T_ErrorSaveContext; + edata->escontext->error_occurred = false; /* * Currently we only support COPY_ON_ERROR_IGNORE. We'll add other * options later */ if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE) - cstate->escontext->details_wanted = false; + edata->escontext->details_wanted = false; } else - cstate->escontext = NULL; + edata->escontext = NULL; /* Convert FORCE_NULL name list to per-column flags, check validity */ cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); @@ -1663,7 +1671,7 @@ BeginCopyFrom(ParseState *pstate, List *attnums; ListCell *cur; - cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); + edata->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.convert_select); @@ -1677,7 +1685,7 @@ BeginCopyFrom(ParseState *pstate, (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), errmsg_internal("selected column \"%s\" not referenced by COPY", NameStr(attr->attname)))); - cstate->convert_select_flags[attnum - 1] = true; + edata->convert_select_flags[attnum - 1] = true; } } @@ -1694,14 +1702,14 @@ BeginCopyFrom(ParseState *pstate, cstate->file_encoding == PG_SQL_ASCII || GetDatabaseEncoding() == PG_SQL_ASCII) { - cstate->need_transcoding = false; + cstate->edata->need_transcoding = false; } else { - cstate->need_transcoding = true; - cstate->conversion_proc = FindDefaultConversionProc(cstate->file_encoding, - GetDatabaseEncoding()); - if (!OidIsValid(cstate->conversion_proc)) + cstate->edata->need_transcoding = true; + cstate->edata->conversion_proc = FindDefaultConversionProc(cstate->file_encoding, + GetDatabaseEncoding()); + if (!OidIsValid(cstate->edata->conversion_proc)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", @@ -1709,17 +1717,17 @@ BeginCopyFrom(ParseState *pstate, pg_encoding_to_char(GetDatabaseEncoding())))); } - cstate->copy_src = COPY_FILE; /* default */ + edata->copy_src = COPY_FILE; /* default */ - cstate->whereClause = whereClause; + edata->whereClause = whereClause; /* Initialize state variables */ - cstate->eol_type = EOL_UNKNOWN; - cstate->cur_relname = RelationGetRelationName(cstate->rel); - cstate->cur_lineno = 0; - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; - cstate->relname_only = false; + edata->eol_type = EOL_UNKNOWN; + edata->cur_relname = RelationGetRelationName(cstate->rel); + edata->cur_lineno = 0; + edata->cur_attname = NULL; + edata->cur_attval = NULL; + edata->relname_only = false; /* * Allocate buffers for the input pipeline. @@ -1727,17 +1735,17 @@ BeginCopyFrom(ParseState *pstate, * attribute_buf and raw_buf are used in both text and binary modes, but * input_buf and line_buf only in text mode. */ - cstate->raw_buf = palloc(RAW_BUF_SIZE + 1); - cstate->raw_buf_index = cstate->raw_buf_len = 0; - cstate->raw_reached_eof = false; + edata->raw_buf = palloc(RAW_BUF_SIZE + 1); + edata->raw_buf_index = edata->raw_buf_len = 0; + edata->raw_reached_eof = false; - initStringInfo(&cstate->attribute_buf); + initStringInfo(&edata->attribute_buf); /* Assign range table and rteperminfos, we'll need them in CopyFrom. */ if (pstate) { - cstate->range_table = pstate->p_rtable; - cstate->rteperminfos = pstate->p_rteperminfos; + edata->range_table = pstate->p_rtable; + edata->rteperminfos = pstate->p_rteperminfos; } num_defaults = 0; @@ -1818,26 +1826,26 @@ BeginCopyFrom(ParseState *pstate, } } - cstate->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool)); + edata->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool)); /* initialize progress */ pgstat_progress_start_command(PROGRESS_COMMAND_COPY, cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid); - cstate->bytes_processed = 0; + edata->bytes_processed = 0; /* We keep those variables in cstate. */ cstate->in_functions = in_functions; cstate->typioparams = typioparams; - cstate->defmap = defmap; - cstate->defexprs = defexprs; - cstate->volatile_defexprs = volatile_defexprs; - cstate->num_defaults = num_defaults; + edata->defmap = defmap; + edata->defexprs = defexprs; + edata->volatile_defexprs = volatile_defexprs; + edata->num_defaults = num_defaults; cstate->is_program = is_program; if (data_source_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_src = COPY_CALLBACK; + edata->copy_src = COPY_CALLBACK; cstate->data_source_cb = data_source_cb; } else if (pipe) @@ -1847,7 +1855,7 @@ BeginCopyFrom(ParseState *pstate, if (whereToSendOutput == DestRemote) ReceiveCopyBegin(cstate); else - cstate->copy_file = stdin; + edata->copy_file = stdin; } else { @@ -1856,8 +1864,8 @@ BeginCopyFrom(ParseState *pstate, if (cstate->is_program) { progress_vals[1] = PROGRESS_COPY_TYPE_PROGRAM; - cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R); - if (cstate->copy_file == NULL) + edata->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R); + if (edata->copy_file == NULL) ereport(ERROR, (errcode_for_file_access(), errmsg("could not execute command \"%s\": %m", @@ -1868,8 +1876,8 @@ BeginCopyFrom(ParseState *pstate, struct stat st; progress_vals[1] = PROGRESS_COPY_TYPE_FILE; - cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R); - if (cstate->copy_file == NULL) + edata->copy_file = AllocateFile(cstate->filename, PG_BINARY_R); + if (edata->copy_file == NULL) { /* copy errno because ereport subfunctions might change it */ int save_errno = errno; @@ -1883,7 +1891,7 @@ BeginCopyFrom(ParseState *pstate, "You may want a client-side facility such as psql's \\copy.") : 0)); } - if (fstat(fileno(cstate->copy_file), &st)) + if (fstat(fileno(edata->copy_file), &st)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", @@ -1923,7 +1931,7 @@ EndCopyFrom(CopyFromState cstate) } else { - if (cstate->filename != NULL && FreeFile(cstate->copy_file)) + if (cstate->filename != NULL && FreeFile(cstate->edata->copy_file)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", @@ -1932,7 +1940,7 @@ EndCopyFrom(CopyFromState cstate) pgstat_progress_end_command(); - MemoryContextDelete(cstate->copycontext); + MemoryContextDelete(cstate->edata->copycontext); pfree(cstate); } @@ -1946,7 +1954,7 @@ ClosePipeFromProgram(CopyFromState cstate) Assert(cstate->is_program); - pclose_rc = ClosePipeStream(cstate->copy_file); + pclose_rc = ClosePipeStream(cstate->edata->copy_file); if (pclose_rc == -1) ereport(ERROR, (errcode_for_file_access(), @@ -1959,7 +1967,7 @@ ClosePipeFromProgram(CopyFromState cstate) * should not report that as an error. Otherwise, SIGPIPE indicates a * problem. */ - if (!cstate->raw_reached_eof && + if (!cstate->edata->raw_reached_eof && wait_result_is_signal(pclose_rc, SIGPIPE)) return; diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index b1ae97b833d..595dc84b172 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -126,12 +126,12 @@ if (1) \ #define REFILL_LINEBUF \ if (1) \ { \ - if (input_buf_ptr > cstate->input_buf_index) \ + if (input_buf_ptr > edata->input_buf_index) \ { \ - appendBinaryStringInfo(&cstate->line_buf, \ - cstate->input_buf + cstate->input_buf_index, \ - input_buf_ptr - cstate->input_buf_index); \ - cstate->input_buf_index = input_buf_ptr; \ + appendBinaryStringInfo(&edata->line_buf, \ + edata->input_buf + edata->input_buf_index, \ + input_buf_ptr - edata->input_buf_index); \ + edata->input_buf_index = input_buf_ptr; \ } \ } else ((void) 0) @@ -180,8 +180,8 @@ ReceiveCopyBegin(CopyFromState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_src = COPY_FRONTEND; - cstate->fe_msgbuf = makeStringInfo(); + cstate->edata->copy_src = COPY_FRONTEND; + cstate->edata->fe_msgbuf = makeStringInfo(); /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); } @@ -246,23 +246,23 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) { int bytesread = 0; - switch (cstate->copy_src) + switch (cstate->edata->copy_src) { case COPY_FILE: - bytesread = fread(databuf, 1, maxread, cstate->copy_file); - if (ferror(cstate->copy_file)) + bytesread = fread(databuf, 1, maxread, cstate->edata->copy_file); + if (ferror(cstate->edata->copy_file)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); if (bytesread == 0) - cstate->raw_reached_eof = true; + cstate->edata->raw_reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread < minread && !cstate->edata->raw_reached_eof) { int avail; - while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len) + while (cstate->edata->fe_msgbuf->cursor >= cstate->edata->fe_msgbuf->len) { /* Try to receive another message */ int mtype; @@ -297,7 +297,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; } /* Now collect the message body */ - if (pq_getmessage(cstate->fe_msgbuf, maxmsglen)) + if (pq_getmessage(cstate->edata->fe_msgbuf, maxmsglen)) ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("unexpected EOF on client connection with an open transaction"))); @@ -309,13 +309,13 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case PqMsg_CopyDone: /* COPY IN correctly terminated by frontend */ - cstate->raw_reached_eof = true; + cstate->edata->raw_reached_eof = true; return bytesread; case PqMsg_CopyFail: ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), errmsg("COPY from stdin failed: %s", - pq_getmsgstring(cstate->fe_msgbuf)))); + pq_getmsgstring(cstate->edata->fe_msgbuf)))); break; case PqMsg_Flush: case PqMsg_Sync: @@ -331,10 +331,10 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) Assert(false); /* NOT REACHED */ } } - avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor; + avail = cstate->edata->fe_msgbuf->len - cstate->edata->fe_msgbuf->cursor; if (avail > maxread) avail = maxread; - pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail); + pq_copymsgbytes(cstate->edata->fe_msgbuf, databuf, avail); databuf = (void *) ((char *) databuf + avail); maxread -= avail; bytesread += avail; @@ -399,12 +399,14 @@ CopyGetInt16(CopyFromState cstate, int16 *val) static void CopyConvertBuf(CopyFromState cstate) { + CopyFromExecutionData *edata = cstate->edata; + /* * If the file and server encoding are the same, no encoding conversion is * required. However, we still need to verify that the input is valid for * the encoding. */ - if (!cstate->need_transcoding) + if (!edata->need_transcoding) { /* * When conversion is not required, input_buf and raw_buf are the @@ -412,8 +414,8 @@ CopyConvertBuf(CopyFromState cstate) * input_buf_len tracks how many of those bytes have already been * verified. */ - int preverifiedlen = cstate->input_buf_len; - int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len; + int preverifiedlen = edata->input_buf_len; + int unverifiedlen = edata->raw_buf_len - edata->input_buf_len; int nverified; if (unverifiedlen == 0) @@ -421,8 +423,8 @@ CopyConvertBuf(CopyFromState cstate) /* * If no more raw data is coming, report the EOF to the caller. */ - if (cstate->raw_reached_eof) - cstate->input_reached_eof = true; + if (edata->raw_reached_eof) + edata->input_reached_eof = true; return; } @@ -431,7 +433,7 @@ CopyConvertBuf(CopyFromState cstate) * previous round. */ nverified = pg_encoding_verifymbstr(cstate->file_encoding, - cstate->raw_buf + preverifiedlen, + edata->raw_buf + preverifiedlen, unverifiedlen); if (nverified == 0) { @@ -444,11 +446,11 @@ CopyConvertBuf(CopyFromState cstate) * least one character, and a failure to do so means that we've * hit an invalid byte sequence. */ - if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding)) - cstate->input_reached_error = true; + if (edata->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding)) + edata->input_reached_error = true; return; } - cstate->input_buf_len += nverified; + edata->input_buf_len += nverified; } else { @@ -462,31 +464,31 @@ CopyConvertBuf(CopyFromState cstate) int dstlen; int convertedlen; - if (RAW_BUF_BYTES(cstate) == 0) + if (RAW_BUF_BYTES(edata) == 0) { /* * If no more raw data is coming, report the EOF to the caller. */ - if (cstate->raw_reached_eof) - cstate->input_reached_eof = true; + if (edata->raw_reached_eof) + edata->input_reached_eof = true; return; } /* * First, copy down any unprocessed data. */ - nbytes = INPUT_BUF_BYTES(cstate); - if (nbytes > 0 && cstate->input_buf_index > 0) - memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index, + nbytes = INPUT_BUF_BYTES(edata); + if (nbytes > 0 && edata->input_buf_index > 0) + memmove(edata->input_buf, edata->input_buf + edata->input_buf_index, nbytes); - cstate->input_buf_index = 0; - cstate->input_buf_len = nbytes; - cstate->input_buf[nbytes] = '\0'; + edata->input_buf_index = 0; + edata->input_buf_len = nbytes; + edata->input_buf[nbytes] = '\0'; - src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index; - srclen = cstate->raw_buf_len - cstate->raw_buf_index; - dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len; - dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1; + src = (unsigned char *) edata->raw_buf + edata->raw_buf_index; + srclen = edata->raw_buf_len - edata->raw_buf_index; + dst = (unsigned char *) edata->input_buf + edata->input_buf_len; + dstlen = INPUT_BUF_SIZE - edata->input_buf_len + 1; /* * Do the conversion. This might stop short, if there is an invalid @@ -501,7 +503,7 @@ CopyConvertBuf(CopyFromState cstate) * after the end-of-input marker as long as it's valid for the * encoding, but that's harmless. */ - convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc, + convertedlen = pg_do_encoding_conversion_buf(cstate->edata->conversion_proc, cstate->file_encoding, GetDatabaseEncoding(), src, srclen, @@ -517,12 +519,12 @@ CopyConvertBuf(CopyFromState cstate) * failure to do so must mean that we've hit a byte sequence * that's invalid. */ - if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH) - cstate->input_reached_error = true; + if (edata->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH) + edata->input_reached_error = true; return; } - cstate->raw_buf_index += convertedlen; - cstate->input_buf_len += strlen((char *) dst); + edata->raw_buf_index += convertedlen; + edata->input_buf_len += strlen((char *) dst); } } @@ -532,18 +534,20 @@ CopyConvertBuf(CopyFromState cstate) static void CopyConversionError(CopyFromState cstate) { - Assert(cstate->raw_buf_len > 0); - Assert(cstate->input_reached_error); + CopyFromExecutionData *edata = cstate->edata; + + Assert(edata->raw_buf_len > 0); + Assert(edata->input_reached_error); - if (!cstate->need_transcoding) + if (!edata->need_transcoding) { /* * Everything up to input_buf_len was successfully verified, and * input_buf_len points to the invalid or incomplete character. */ report_invalid_encoding(cstate->file_encoding, - cstate->raw_buf + cstate->input_buf_len, - cstate->raw_buf_len - cstate->input_buf_len); + edata->raw_buf + edata->input_buf_len, + edata->raw_buf_len - edata->input_buf_len); } else { @@ -560,12 +564,12 @@ CopyConversionError(CopyFromState cstate) unsigned char *dst; int dstlen; - src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index; - srclen = cstate->raw_buf_len - cstate->raw_buf_index; - dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len; - dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1; + src = (unsigned char *) edata->raw_buf + edata->raw_buf_index; + srclen = edata->raw_buf_len - edata->raw_buf_index; + dst = (unsigned char *) edata->input_buf + edata->input_buf_len; + dstlen = INPUT_BUF_SIZE - edata->input_buf_len + 1; - (void) pg_do_encoding_conversion_buf(cstate->conversion_proc, + (void) pg_do_encoding_conversion_buf(cstate->edata->conversion_proc, cstate->file_encoding, GetDatabaseEncoding(), src, srclen, @@ -589,6 +593,7 @@ CopyConversionError(CopyFromState cstate) static void CopyLoadRawBuf(CopyFromState cstate) { + CopyFromExecutionData *edata = cstate->edata; int nbytes; int inbytes; @@ -596,45 +601,45 @@ CopyLoadRawBuf(CopyFromState cstate) * In text mode, if encoding conversion is not required, raw_buf and * input_buf point to the same buffer. Their len/index better agree, too. */ - if (cstate->raw_buf == cstate->input_buf) + if (edata->raw_buf == edata->input_buf) { - Assert(!cstate->need_transcoding); - Assert(cstate->raw_buf_index == cstate->input_buf_index); - Assert(cstate->input_buf_len <= cstate->raw_buf_len); + Assert(!edata->need_transcoding); + Assert(edata->raw_buf_index == edata->input_buf_index); + Assert(edata->input_buf_len <= edata->raw_buf_len); } /* * Copy down the unprocessed data if any. */ - nbytes = RAW_BUF_BYTES(cstate); - if (nbytes > 0 && cstate->raw_buf_index > 0) - memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index, + nbytes = RAW_BUF_BYTES(edata); + if (nbytes > 0 && edata->raw_buf_index > 0) + memmove(edata->raw_buf, edata->raw_buf + edata->raw_buf_index, nbytes); - cstate->raw_buf_len -= cstate->raw_buf_index; - cstate->raw_buf_index = 0; + edata->raw_buf_len -= edata->raw_buf_index; + edata->raw_buf_index = 0; /* * If raw_buf and input_buf are in fact the same buffer, adjust the * input_buf variables, too. */ - if (cstate->raw_buf == cstate->input_buf) + if (edata->raw_buf == edata->input_buf) { - cstate->input_buf_len -= cstate->input_buf_index; - cstate->input_buf_index = 0; + edata->input_buf_len -= edata->input_buf_index; + edata->input_buf_index = 0; } /* Load more data */ - inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, - 1, RAW_BUF_SIZE - cstate->raw_buf_len); + inbytes = CopyGetData(cstate, edata->raw_buf + edata->raw_buf_len, + 1, RAW_BUF_SIZE - edata->raw_buf_len); nbytes += inbytes; - cstate->raw_buf[nbytes] = '\0'; - cstate->raw_buf_len = nbytes; + edata->raw_buf[nbytes] = '\0'; + edata->raw_buf_len = nbytes; - cstate->bytes_processed += inbytes; - pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); + edata->bytes_processed += inbytes; + pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, edata->bytes_processed); if (inbytes == 0) - cstate->raw_reached_eof = true; + edata->raw_reached_eof = true; } /* @@ -643,24 +648,25 @@ CopyLoadRawBuf(CopyFromState cstate) * On return, at least one more input character is loaded into * input_buf, or input_reached_eof is set. * - * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start + * If INPUT_BUF_BYTES(edata) > 0, the unprocessed bytes are moved to the start * of the buffer and then we load more data after that. */ static void CopyLoadInputBuf(CopyFromState cstate) { - int nbytes = INPUT_BUF_BYTES(cstate); + CopyFromExecutionData *edata = cstate->edata; + int nbytes = INPUT_BUF_BYTES(edata); /* * The caller has updated input_buf_index to indicate how much of the * input has been consumed and isn't needed anymore. If input_buf is the * same physical area as raw_buf, update raw_buf_index accordingly. */ - if (cstate->raw_buf == cstate->input_buf) + if (edata->raw_buf == edata->input_buf) { - Assert(!cstate->need_transcoding); - Assert(cstate->input_buf_index >= cstate->raw_buf_index); - cstate->raw_buf_index = cstate->input_buf_index; + Assert(!edata->need_transcoding); + Assert(edata->input_buf_index >= edata->raw_buf_index); + edata->raw_buf_index = edata->input_buf_index; } for (;;) @@ -669,7 +675,7 @@ CopyLoadInputBuf(CopyFromState cstate) CopyConvertBuf(cstate); /* If we now have some more input bytes ready, return them */ - if (INPUT_BUF_BYTES(cstate) > nbytes) + if (INPUT_BUF_BYTES(edata) > nbytes) return; /* @@ -677,15 +683,15 @@ CopyLoadInputBuf(CopyFromState cstate) * multi-byte character but there is no more raw input data, report * conversion error. */ - if (cstate->input_reached_error) + if (edata->input_reached_error) CopyConversionError(cstate); /* no more input, and everything has been converted */ - if (cstate->input_reached_eof) + if (edata->input_reached_eof) break; /* Try to load more raw data */ - Assert(!cstate->raw_reached_eof); + Assert(!edata->raw_reached_eof); CopyLoadRawBuf(cstate); } } @@ -700,13 +706,14 @@ CopyLoadInputBuf(CopyFromState cstate) static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) { + CopyFromExecutionData *edata = cstate->edata; int copied_bytes = 0; - if (RAW_BUF_BYTES(cstate) >= nbytes) + if (RAW_BUF_BYTES(edata) >= nbytes) { /* Enough bytes are present in the buffer. */ - memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes); - cstate->raw_buf_index += nbytes; + memcpy(dest, edata->raw_buf + edata->raw_buf_index, nbytes); + edata->raw_buf_index += nbytes; copied_bytes = nbytes; } else @@ -720,17 +727,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) int copy_bytes; /* Load more data if buffer is empty. */ - if (RAW_BUF_BYTES(cstate) == 0) + if (RAW_BUF_BYTES(edata) == 0) { CopyLoadRawBuf(cstate); - if (cstate->raw_reached_eof) + if (edata->raw_reached_eof) break; /* EOF */ } /* Transfer some bytes. */ - copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate)); - memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes); - cstate->raw_buf_index += copy_bytes; + copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(edata)); + memcpy(dest, edata->raw_buf + edata->raw_buf_index, copy_bytes); + edata->raw_buf_index += copy_bytes; dest += copy_bytes; copied_bytes += copy_bytes; } while (copied_bytes < nbytes); @@ -770,6 +777,7 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv) { + CopyFromExecutionData *edata = cstate->edata; int fldct; bool done = false; @@ -777,7 +785,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields Assert(!cstate->opts.binary); /* on input check that the header line is correct if needed */ - if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE) + if (edata->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE) { ListCell *cur; TupleDesc tupDesc; @@ -791,7 +799,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields for (int i = 0; i < lines_to_skip; i++) { - cstate->cur_lineno++; + edata->cur_lineno++; if ((done = CopyReadLine(cstate, is_csv))) break; } @@ -818,9 +826,9 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields char *colName; Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - Assert(fldnum < cstate->max_fields); + Assert(fldnum < edata->max_fields); - colName = cstate->raw_fields[fldnum++]; + colName = edata->raw_fields[fldnum++]; if (colName == NULL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -841,7 +849,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields return false; } - cstate->cur_lineno++; + edata->cur_lineno++; /* Actually read the line into memory here */ done = CopyReadLine(cstate, is_csv); @@ -851,7 +859,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields * characters, we act as though it was newline followed by EOF, ie, * process the line and then exit loop on next iteration. */ - if (done && cstate->line_buf.len == 0) + if (done && edata->line_buf.len == 0) return false; /* Parse the line into de-escaped field values */ @@ -860,7 +868,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields else fldct = CopyReadAttributesText(cstate); - *fields = cstate->raw_fields; + *fields = edata->raw_fields; *nfields = fldct; return true; } @@ -882,10 +890,10 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, { TupleDesc tupDesc; AttrNumber num_phys_attrs, - num_defaults = cstate->num_defaults; + num_defaults = cstate->edata->num_defaults; int i; - int *defmap = cstate->defmap; - ExprState **defexprs = cstate->defexprs; + int *defmap = cstate->edata->defmap; + ExprState **defexprs = cstate->edata->defexprs; tupDesc = RelationGetDescr(cstate->rel); num_phys_attrs = tupDesc->natts; @@ -893,7 +901,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, /* Initialize all values for row to NULL */ MemSet(values, 0, num_phys_attrs * sizeof(Datum)); MemSet(nulls, true, num_phys_attrs * sizeof(bool)); - MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); + MemSet(cstate->edata->defaults, false, num_phys_attrs * sizeof(bool)); /* Get one row from source */ if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls)) @@ -946,11 +954,12 @@ static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls, bool is_csv) { + CopyFromExecutionData *edata = cstate->edata; TupleDesc tupDesc; AttrNumber attr_count; FmgrInfo *in_functions = cstate->in_functions; Oid *typioparams = cstate->typioparams; - ExprState **defexprs = cstate->defexprs; + ExprState **defexprs = edata->defexprs; char **field_strings; ListCell *cur; int fldct; @@ -986,8 +995,8 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, NameStr(att->attname)))); string = field_strings[fieldno++]; - if (cstate->convert_select_flags && - !cstate->convert_select_flags[m]) + if (edata->convert_select_flags && + !edata->convert_select_flags[m]) { /* ignore input field, leaving column as NULL */ continue; @@ -1017,13 +1026,13 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, } } - cstate->cur_attname = NameStr(att->attname); - cstate->cur_attval = string; + edata->cur_attname = NameStr(att->attname); + edata->cur_attval = string; if (string != NULL) nulls[m] = false; - if (cstate->defaults[m]) + if (edata->defaults[m]) { /* We must have switched into the per-tuple memory context */ Assert(econtext != NULL); @@ -1039,12 +1048,12 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, string, typioparams[m], att->atttypmod, - (Node *) cstate->escontext, + (Node *) edata->escontext, &values[m])) { Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); - cstate->num_errors++; + edata->num_errors++; if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) { @@ -1053,36 +1062,36 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, * notice message, we suppress error context information other * than the relation name. */ - Assert(!cstate->relname_only); - cstate->relname_only = true; + Assert(!edata->relname_only); + edata->relname_only = true; - if (cstate->cur_attval) + if (edata->cur_attval) { char *attval; - attval = CopyLimitPrintoutLength(cstate->cur_attval); + attval = CopyLimitPrintoutLength(edata->cur_attval); ereport(NOTICE, errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", - cstate->cur_lineno, - cstate->cur_attname, + edata->cur_lineno, + edata->cur_attname, attval)); pfree(attval); } else ereport(NOTICE, errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", - cstate->cur_lineno, - cstate->cur_attname)); + edata->cur_lineno, + edata->cur_attname)); /* reset relname_only */ - cstate->relname_only = false; + edata->relname_only = false; } return true; } - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; + edata->cur_attname = NULL; + edata->cur_attval = NULL; } Assert(fieldno == attr_count); @@ -1105,7 +1114,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, tupDesc = RelationGetDescr(cstate->rel); attr_count = list_length(cstate->attnumlist); - cstate->cur_lineno++; + cstate->edata->cur_lineno++; if (!CopyGetInt16(cstate, &fld_count)) { @@ -1144,13 +1153,13 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, int m = attnum - 1; Form_pg_attribute att = TupleDescAttr(tupDesc, m); - cstate->cur_attname = NameStr(att->attname); + cstate->edata->cur_attname = NameStr(att->attname); values[m] = CopyReadBinaryAttribute(cstate, &in_functions[m], typioparams[m], att->atttypmod, &nulls[m]); - cstate->cur_attname = NULL; + cstate->edata->cur_attname = NULL; } return true; @@ -1166,10 +1175,11 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, static bool CopyReadLine(CopyFromState cstate, bool is_csv) { + CopyFromExecutionData *edata = cstate->edata; bool result; - resetStringInfo(&cstate->line_buf); - cstate->line_buf_valid = false; + resetStringInfo(&edata->line_buf); + edata->line_buf_valid = false; /* Parse data and transfer into line_buf */ result = CopyReadLineText(cstate, is_csv); @@ -1181,19 +1191,19 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (edata->copy_src == COPY_FRONTEND) { int inbytes; do { - inbytes = CopyGetData(cstate, cstate->input_buf, + inbytes = CopyGetData(cstate, edata->input_buf, 1, INPUT_BUF_SIZE); } while (inbytes > 0); - cstate->input_buf_index = 0; - cstate->input_buf_len = 0; - cstate->raw_buf_index = 0; - cstate->raw_buf_len = 0; + edata->input_buf_index = 0; + edata->input_buf_len = 0; + edata->raw_buf_index = 0; + edata->raw_buf_len = 0; } } else @@ -1202,26 +1212,26 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * If we didn't hit EOF, then we must have transferred the EOL marker * to line_buf along with the data. Get rid of it. */ - switch (cstate->eol_type) + switch (edata->eol_type) { case EOL_NL: - Assert(cstate->line_buf.len >= 1); - Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n'); - cstate->line_buf.len--; - cstate->line_buf.data[cstate->line_buf.len] = '\0'; + Assert(edata->line_buf.len >= 1); + Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\n'); + edata->line_buf.len--; + edata->line_buf.data[edata->line_buf.len] = '\0'; break; case EOL_CR: - Assert(cstate->line_buf.len >= 1); - Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r'); - cstate->line_buf.len--; - cstate->line_buf.data[cstate->line_buf.len] = '\0'; + Assert(edata->line_buf.len >= 1); + Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\r'); + edata->line_buf.len--; + edata->line_buf.data[edata->line_buf.len] = '\0'; break; case EOL_CRNL: - Assert(cstate->line_buf.len >= 2); - Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r'); - Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n'); - cstate->line_buf.len -= 2; - cstate->line_buf.data[cstate->line_buf.len] = '\0'; + Assert(edata->line_buf.len >= 2); + Assert(edata->line_buf.data[edata->line_buf.len - 2] == '\r'); + Assert(edata->line_buf.data[edata->line_buf.len - 1] == '\n'); + edata->line_buf.len -= 2; + edata->line_buf.data[edata->line_buf.len] = '\0'; break; case EOL_UNKNOWN: /* shouldn't get here */ @@ -1231,7 +1241,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) } /* Now it's safe to use the buffer in error messages */ - cstate->line_buf_valid = true; + edata->line_buf_valid = true; return result; } @@ -1242,6 +1252,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) static bool CopyReadLineText(CopyFromState cstate, bool is_csv) { + CopyFromExecutionData *edata = cstate->edata; char *copy_input_buf; int input_buf_ptr; int copy_buf_len; @@ -1289,9 +1300,9 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) * For a little extra speed within the loop, we copy input_buf and * input_buf_len into local variables. */ - copy_input_buf = cstate->input_buf; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + copy_input_buf = edata->input_buf; + input_buf_ptr = edata->input_buf_index; + copy_buf_len = edata->input_buf_len; for (;;) { @@ -1312,15 +1323,15 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) CopyLoadInputBuf(cstate); /* update our local variables */ - hit_eof = cstate->input_reached_eof; - input_buf_ptr = cstate->input_buf_index; - copy_buf_len = cstate->input_buf_len; + hit_eof = edata->input_reached_eof; + input_buf_ptr = edata->input_buf_index; + copy_buf_len = edata->input_buf_len; /* * If we are completely out of data, break out of the loop, * reporting EOF. */ - if (INPUT_BUF_BYTES(cstate) <= 0) + if (INPUT_BUF_BYTES(edata) <= 0) { result = true; break; @@ -1366,16 +1377,16 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) * best we can do. (XXX it's arguable whether we should do this * at all --- is cur_lineno a physical or logical count?) */ - if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r')) - cstate->cur_lineno++; + if (in_quote && c == (edata->eol_type == EOL_NL ? '\n' : '\r')) + edata->cur_lineno++; } /* Process \r */ if (c == '\r' && (!is_csv || !in_quote)) { /* Check for \r\n on first line, _and_ handle \r\n. */ - if (cstate->eol_type == EOL_UNKNOWN || - cstate->eol_type == EOL_CRNL) + if (edata->eol_type == EOL_UNKNOWN || + edata->eol_type == EOL_CRNL) { /* * If need more data, go back to loop top to load it. @@ -1391,12 +1402,12 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) if (c == '\n') { input_buf_ptr++; /* eat newline */ - cstate->eol_type = EOL_CRNL; /* in case not set yet */ + edata->eol_type = EOL_CRNL; /* in case not set yet */ } else { /* found \r, but no \n */ - if (cstate->eol_type == EOL_CRNL) + if (edata->eol_type == EOL_CRNL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), !is_csv ? @@ -1410,10 +1421,10 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) * if we got here, it is the first line and we didn't find * \n, so don't consume the peeked character */ - cstate->eol_type = EOL_CR; + edata->eol_type = EOL_CR; } } - else if (cstate->eol_type == EOL_NL) + else if (edata->eol_type == EOL_NL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), !is_csv ? @@ -1429,7 +1440,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) /* Process \n */ if (c == '\n' && (!is_csv || !in_quote)) { - if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL) + if (edata->eol_type == EOL_CR || edata->eol_type == EOL_CRNL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), !is_csv ? @@ -1438,7 +1449,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) !is_csv ? errhint("Use \"\\n\" to represent newline.") : errhint("Use quoted CSV field to represent newline."))); - cstate->eol_type = EOL_NL; /* in case not set yet */ + edata->eol_type = EOL_NL; /* in case not set yet */ /* If reach here, we have found the line terminator */ break; } @@ -1465,7 +1476,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) if (c2 == '.') { input_buf_ptr++; /* consume the '.' */ - if (cstate->eol_type == EOL_CRNL) + if (edata->eol_type == EOL_CRNL) { /* Get the next character */ IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); @@ -1492,9 +1503,9 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("end-of-copy marker is not alone on its line"))); - if ((cstate->eol_type == EOL_NL && c2 != '\n') || - (cstate->eol_type == EOL_CRNL && c2 != '\n') || - (cstate->eol_type == EOL_CR && c2 != '\r')) + if ((edata->eol_type == EOL_NL && c2 != '\n') || + (edata->eol_type == EOL_CRNL && c2 != '\n') || + (edata->eol_type == EOL_CR && c2 != '\r')) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("end-of-copy marker does not match previous newline style"))); @@ -1502,8 +1513,8 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) /* * If there is any data on this line before the \., complain. */ - if (cstate->line_buf.len > 0 || - prev_raw_ptr > cstate->input_buf_index) + if (edata->line_buf.len > 0 || + prev_raw_ptr > edata->input_buf_index) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("end-of-copy marker is not alone on its line"))); @@ -1511,7 +1522,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) /* * Discard the \. and newline, then report EOF. */ - cstate->input_buf_index = input_buf_ptr; + edata->input_buf_index = input_buf_ptr; result = true; /* report EOF */ break; } @@ -1572,6 +1583,7 @@ GetDecimalFromHex(char hex) static int CopyReadAttributesText(CopyFromState cstate) { + CopyFromExecutionData *edata = cstate->edata; char delimc = cstate->opts.delim[0]; int fieldno; char *output_ptr; @@ -1582,31 +1594,31 @@ CopyReadAttributesText(CopyFromState cstate) * We need a special case for zero-column tables: check that the input * line is empty, and return. */ - if (cstate->max_fields <= 0) + if (edata->max_fields <= 0) { - if (cstate->line_buf.len != 0) + if (edata->line_buf.len != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("extra data after last expected column"))); return 0; } - resetStringInfo(&cstate->attribute_buf); + resetStringInfo(&edata->attribute_buf); /* * The de-escaped attributes will certainly not be longer than the input * data line, so we can just force attribute_buf to be large enough and * then transfer data without any checks for enough space. We need to do * it this way because enlarging attribute_buf mid-stream would invalidate - * pointers already stored into cstate->raw_fields[]. + * pointers already stored into edata->raw_fields[]. */ - if (cstate->attribute_buf.maxlen <= cstate->line_buf.len) - enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len); - output_ptr = cstate->attribute_buf.data; + if (edata->attribute_buf.maxlen <= edata->line_buf.len) + enlargeStringInfo(&edata->attribute_buf, edata->line_buf.len); + output_ptr = edata->attribute_buf.data; /* set pointer variables for loop */ - cur_ptr = cstate->line_buf.data; - line_end_ptr = cstate->line_buf.data + cstate->line_buf.len; + cur_ptr = edata->line_buf.data; + line_end_ptr = edata->line_buf.data + edata->line_buf.len; /* Outer loop iterates over fields */ fieldno = 0; @@ -1619,16 +1631,16 @@ CopyReadAttributesText(CopyFromState cstate) bool saw_non_ascii = false; /* Make sure there is enough space for the next value */ - if (fieldno >= cstate->max_fields) + if (fieldno >= edata->max_fields) { - cstate->max_fields *= 2; - cstate->raw_fields = - repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *)); + edata->max_fields *= 2; + edata->raw_fields = + repalloc(edata->raw_fields, edata->max_fields * sizeof(char *)); } /* Remember start of field on both input and output sides */ start_ptr = cur_ptr; - cstate->raw_fields[fieldno] = output_ptr; + edata->raw_fields[fieldno] = output_ptr; /* * Scan data for field. @@ -1757,7 +1769,7 @@ CopyReadAttributesText(CopyFromState cstate) input_len = end_ptr - start_ptr; if (input_len == cstate->opts.null_print_len && strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) - cstate->raw_fields[fieldno] = NULL; + edata->raw_fields[fieldno] = NULL; /* Check whether raw input matched default marker */ else if (fieldno < list_length(cstate->attnumlist) && cstate->opts.default_print && @@ -1767,10 +1779,10 @@ CopyReadAttributesText(CopyFromState cstate) /* fieldno is 0-indexed and attnum is 1-indexed */ int m = list_nth_int(cstate->attnumlist, fieldno) - 1; - if (cstate->defexprs[m] != NULL) + if (edata->defexprs[m] != NULL) { /* defaults contain entries for all physical attributes */ - cstate->defaults[m] = true; + edata->defaults[m] = true; } else { @@ -1794,7 +1806,7 @@ CopyReadAttributesText(CopyFromState cstate) */ if (saw_non_ascii) { - char *fld = cstate->raw_fields[fieldno]; + char *fld = edata->raw_fields[fieldno]; pg_verifymbstr(fld, output_ptr - fld, false); } @@ -1812,7 +1824,7 @@ CopyReadAttributesText(CopyFromState cstate) /* Clean up state of attribute_buf */ output_ptr--; Assert(*output_ptr == '\0'); - cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data); + edata->attribute_buf.len = (output_ptr - edata->attribute_buf.data); return fieldno; } @@ -1826,6 +1838,7 @@ CopyReadAttributesText(CopyFromState cstate) static int CopyReadAttributesCSV(CopyFromState cstate) { + CopyFromExecutionData *edata = cstate->edata; char delimc = cstate->opts.delim[0]; char quotec = cstate->opts.quote[0]; char escapec = cstate->opts.escape[0]; @@ -1838,31 +1851,31 @@ CopyReadAttributesCSV(CopyFromState cstate) * We need a special case for zero-column tables: check that the input * line is empty, and return. */ - if (cstate->max_fields <= 0) + if (edata->max_fields <= 0) { - if (cstate->line_buf.len != 0) + if (edata->line_buf.len != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("extra data after last expected column"))); return 0; } - resetStringInfo(&cstate->attribute_buf); + resetStringInfo(&edata->attribute_buf); /* * The de-escaped attributes will certainly not be longer than the input * data line, so we can just force attribute_buf to be large enough and * then transfer data without any checks for enough space. We need to do * it this way because enlarging attribute_buf mid-stream would invalidate - * pointers already stored into cstate->raw_fields[]. + * pointers already stored into edata->raw_fields[]. */ - if (cstate->attribute_buf.maxlen <= cstate->line_buf.len) - enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len); - output_ptr = cstate->attribute_buf.data; + if (edata->attribute_buf.maxlen <= edata->line_buf.len) + enlargeStringInfo(&edata->attribute_buf, edata->line_buf.len); + output_ptr = edata->attribute_buf.data; /* set pointer variables for loop */ - cur_ptr = cstate->line_buf.data; - line_end_ptr = cstate->line_buf.data + cstate->line_buf.len; + cur_ptr = edata->line_buf.data; + line_end_ptr = edata->line_buf.data + edata->line_buf.len; /* Outer loop iterates over fields */ fieldno = 0; @@ -1875,16 +1888,16 @@ CopyReadAttributesCSV(CopyFromState cstate) int input_len; /* Make sure there is enough space for the next value */ - if (fieldno >= cstate->max_fields) + if (fieldno >= edata->max_fields) { - cstate->max_fields *= 2; - cstate->raw_fields = - repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *)); + edata->max_fields *= 2; + edata->raw_fields = + repalloc(edata->raw_fields, edata->max_fields * sizeof(char *)); } /* Remember start of field on both input and output sides */ start_ptr = cur_ptr; - cstate->raw_fields[fieldno] = output_ptr; + edata->raw_fields[fieldno] = output_ptr; /* * Scan data for field, @@ -1972,7 +1985,7 @@ endfield: input_len = end_ptr - start_ptr; if (!saw_quote && input_len == cstate->opts.null_print_len && strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) - cstate->raw_fields[fieldno] = NULL; + edata->raw_fields[fieldno] = NULL; /* Check whether raw input matched default marker */ else if (fieldno < list_length(cstate->attnumlist) && cstate->opts.default_print && @@ -1982,10 +1995,10 @@ endfield: /* fieldno is 0-index and attnum is 1-index */ int m = list_nth_int(cstate->attnumlist, fieldno) - 1; - if (cstate->defexprs[m] != NULL) + if (edata->defexprs[m] != NULL) { /* defaults contain entries for all physical attributes */ - cstate->defaults[m] = true; + edata->defaults[m] = true; } else { @@ -2009,7 +2022,7 @@ endfield: /* Clean up state of attribute_buf */ output_ptr--; Assert(*output_ptr == '\0'); - cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data); + edata->attribute_buf.len = (output_ptr - edata->attribute_buf.data); return fieldno; } @@ -2023,6 +2036,7 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull) { + CopyFromExecutionData *edata = cstate->edata; int32 fld_size; Datum result; @@ -2041,24 +2055,24 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, errmsg("invalid field size"))); /* reset attribute_buf to empty, and load raw data in it */ - resetStringInfo(&cstate->attribute_buf); + resetStringInfo(&edata->attribute_buf); - enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, + enlargeStringInfo(&edata->attribute_buf, fld_size); + if (CopyReadBinaryData(cstate, edata->attribute_buf.data, fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); - cstate->attribute_buf.len = fld_size; - cstate->attribute_buf.data[fld_size] = '\0'; + edata->attribute_buf.len = fld_size; + edata->attribute_buf.data[fld_size] = '\0'; /* Call the column type's binary input converter */ - result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf, + result = ReceiveFunctionCall(flinfo, &edata->attribute_buf, typioparam, typmod); /* Trouble if it didn't eat the whole buffer */ - if (cstate->attribute_buf.cursor != cstate->attribute_buf.len) + if (edata->attribute_buf.cursor != edata->attribute_buf.len) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("incorrect binary data format"))); diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 67b94b91cae..e90fab6d958 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -62,40 +62,24 @@ typedef enum CopyDest * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true * when we have to do it the hard way. */ -typedef struct CopyToStateData +typedef struct CopyToExecutionData { - /* format-specific routines */ - const CopyToRoutine *routine; - /* low-level state data */ CopyDest copy_dest; /* type of copy source/destination */ FILE *copy_file; /* used if copy_dest == COPY_FILE */ StringInfo fe_msgbuf; /* used for all dests during COPY TO */ - int file_encoding; /* file or remote side's character encoding */ bool need_transcoding; /* file encoding diff from server? */ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - /* * Working state */ MemoryContext copycontext; /* per-copy execution context */ - FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; +} CopyToExecutionData; /* DestReceiver for COPY (query) TO */ typedef struct @@ -193,7 +177,7 @@ CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc) * For non-binary copy, we need to convert null_print to file encoding, * because it will be sent directly with CopySendString. */ - if (cstate->need_transcoding) + if (cstate->edata->need_transcoding) cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, cstate->opts.null_print_len, cstate->file_encoding); @@ -401,14 +385,14 @@ SendCopyBegin(CopyToState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->edata->copy_dest = COPY_FRONTEND; } static void SendCopyEnd(CopyToState cstate) { /* Shouldn't have any unsent data */ - Assert(cstate->fe_msgbuf->len == 0); + Assert(cstate->edata->fe_msgbuf->len == 0); /* Send Copy Done message */ pq_putemptymessage(PqMsg_CopyDone); } @@ -426,32 +410,32 @@ SendCopyEnd(CopyToState cstate) static void CopySendData(CopyToState cstate, const void *databuf, int datasize) { - appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize); + appendBinaryStringInfo(cstate->edata->fe_msgbuf, databuf, datasize); } static void CopySendString(CopyToState cstate, const char *str) { - appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str)); + appendBinaryStringInfo(cstate->edata->fe_msgbuf, str, strlen(str)); } static void CopySendChar(CopyToState cstate, char c) { - appendStringInfoCharMacro(cstate->fe_msgbuf, c); + appendStringInfoCharMacro(cstate->edata->fe_msgbuf, c); } static void CopySendEndOfRow(CopyToState cstate) { - StringInfo fe_msgbuf = cstate->fe_msgbuf; + StringInfo fe_msgbuf = cstate->edata->fe_msgbuf; - switch (cstate->copy_dest) + switch (cstate->edata->copy_dest) { case COPY_FILE: if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, - cstate->copy_file) != 1 || - ferror(cstate->copy_file)) + cstate->edata->copy_file) != 1 || + ferror(cstate->edata->copy_file)) { if (cstate->is_program) { @@ -492,8 +476,8 @@ CopySendEndOfRow(CopyToState cstate) } /* Update the progress */ - cstate->bytes_processed += fe_msgbuf->len; - pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); + cstate->edata->bytes_processed += fe_msgbuf->len; + pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->edata->bytes_processed); resetStringInfo(fe_msgbuf); } @@ -505,7 +489,7 @@ CopySendEndOfRow(CopyToState cstate) static inline void CopySendTextLikeEndOfRow(CopyToState cstate) { - switch (cstate->copy_dest) + switch (cstate->edata->copy_dest) { case COPY_FILE: /* Default line termination depends on platform */ @@ -565,7 +549,7 @@ ClosePipeToProgram(CopyToState cstate) Assert(cstate->is_program); - pclose_rc = ClosePipeStream(cstate->copy_file); + pclose_rc = ClosePipeStream(cstate->edata->copy_file); if (pclose_rc == -1) ereport(ERROR, (errcode_for_file_access(), @@ -592,7 +576,7 @@ EndCopy(CopyToState cstate) } else { - if (cstate->filename != NULL && FreeFile(cstate->copy_file)) + if (cstate->filename != NULL && FreeFile(cstate->edata->copy_file)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", @@ -601,7 +585,7 @@ EndCopy(CopyToState cstate) pgstat_progress_end_command(); - MemoryContextDelete(cstate->copycontext); + MemoryContextDelete(cstate->edata->copycontext); pfree(cstate); } @@ -688,16 +672,17 @@ BeginCopyTo(ParseState *pstate, /* Allocate workspace and zero all fields */ cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateData)); + cstate->edata = (CopyToExecutionData *) palloc0(sizeof(CopyToExecutionData)); /* * We allocate everything used by a cstate in a new memory context. This * avoids memory leaks during repeated use of COPY in a query. */ - cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext, - "COPY", - ALLOCSET_DEFAULT_SIZES); + cstate->edata->copycontext = AllocSetContextCreate(CurrentMemoryContext, + "COPY", + ALLOCSET_DEFAULT_SIZES); - oldcontext = MemoryContextSwitchTo(cstate->copycontext); + oldcontext = MemoryContextSwitchTo(cstate->edata->copycontext); /* Extract options from the statement node tree */ ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options); @@ -895,19 +880,19 @@ BeginCopyTo(ParseState *pstate, */ if (cstate->file_encoding == GetDatabaseEncoding() || cstate->file_encoding == PG_SQL_ASCII) - cstate->need_transcoding = false; + cstate->edata->need_transcoding = false; else - cstate->need_transcoding = true; + cstate->edata->need_transcoding = true; /* See Multibyte encoding comment above */ - cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); + cstate->edata->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - cstate->copy_dest = COPY_FILE; /* default */ + cstate->edata->copy_dest = COPY_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->edata->copy_dest = COPY_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) @@ -916,7 +901,7 @@ BeginCopyTo(ParseState *pstate, Assert(!is_program); /* the grammar does not allow this */ if (whereToSendOutput != DestRemote) - cstate->copy_file = stdout; + cstate->edata->copy_file = stdout; } else { @@ -926,8 +911,8 @@ BeginCopyTo(ParseState *pstate, if (is_program) { progress_vals[1] = PROGRESS_COPY_TYPE_PROGRAM; - cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W); - if (cstate->copy_file == NULL) + cstate->edata->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W); + if (cstate->edata->copy_file == NULL) ereport(ERROR, (errcode_for_file_access(), errmsg("could not execute command \"%s\": %m", @@ -952,14 +937,14 @@ BeginCopyTo(ParseState *pstate, oumask = umask(S_IWGRP | S_IWOTH); PG_TRY(); { - cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W); + cstate->edata->copy_file = AllocateFile(cstate->filename, PG_BINARY_W); } PG_FINALLY(); { umask(oumask); } PG_END_TRY(); - if (cstate->copy_file == NULL) + if (cstate->edata->copy_file == NULL) { /* copy errno because ereport subfunctions might change it */ int save_errno = errno; @@ -973,7 +958,7 @@ BeginCopyTo(ParseState *pstate, "You may want a client-side facility such as psql's \\copy.") : 0)); } - if (fstat(fileno(cstate->copy_file), &st)) + if (fstat(fileno(cstate->edata->copy_file), &st)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", @@ -991,7 +976,7 @@ BeginCopyTo(ParseState *pstate, cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid); pgstat_progress_update_multi_param(2, progress_cols, progress_vals); - cstate->bytes_processed = 0; + cstate->edata->bytes_processed = 0; MemoryContextSwitchTo(oldcontext); @@ -1042,8 +1027,8 @@ DoCopyTo(CopyToState cstate) num_phys_attrs = tupDesc->natts; cstate->opts.null_print_client = cstate->opts.null_print; /* default */ - /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ - cstate->fe_msgbuf = makeStringInfo(); + /* We use fe_msgbuf as a per-row buffer regardless of edata->copy_dest */ + cstate->edata->fe_msgbuf = makeStringInfo(); /* Get info about the columns we need to process. */ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); @@ -1062,9 +1047,9 @@ DoCopyTo(CopyToState cstate) * datatype output routines, and should be faster than retail pfree's * anyway. (We don't need a whole econtext as CopyFrom does.) */ - cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext, - "COPY TO", - ALLOCSET_DEFAULT_SIZES); + cstate->edata->rowcontext = AllocSetContextCreate(CurrentMemoryContext, + "COPY TO", + ALLOCSET_DEFAULT_SIZES); cstate->routine->CopyToStart(cstate, tupDesc); @@ -1107,7 +1092,7 @@ DoCopyTo(CopyToState cstate) cstate->routine->CopyToEnd(cstate); - MemoryContextDelete(cstate->rowcontext); + MemoryContextDelete(cstate->edata->rowcontext); if (fe_copy) SendCopyEnd(cstate); @@ -1123,8 +1108,8 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) { MemoryContext oldcontext; - MemoryContextReset(cstate->rowcontext); - oldcontext = MemoryContextSwitchTo(cstate->rowcontext); + MemoryContextReset(cstate->edata->rowcontext); + oldcontext = MemoryContextSwitchTo(cstate->edata->rowcontext); /* Make sure the tuple is fully deconstructed */ slot_getallattrs(slot); @@ -1151,7 +1136,7 @@ CopyAttributeOutText(CopyToState cstate, const char *string) char c; char delimc = cstate->opts.delim[0]; - if (cstate->need_transcoding) + if (cstate->edata->need_transcoding) ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); else ptr = string; @@ -1170,7 +1155,7 @@ CopyAttributeOutText(CopyToState cstate, const char *string) * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out * of the normal safe-encoding path. */ - if (cstate->encoding_embeds_ascii) + if (cstate->edata->encoding_embeds_ascii) { start = ptr; while ((c = *ptr) != '\0') @@ -1312,7 +1297,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, if (!use_quote && strcmp(string, cstate->opts.null_print) == 0) use_quote = true; - if (cstate->need_transcoding) + if (cstate->edata->need_transcoding) ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); else ptr = string; @@ -1342,7 +1327,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, use_quote = true; break; } - if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) + if (IS_HIGHBIT_SET(c) && cstate->edata->encoding_embeds_ascii) tptr += pg_encoding_mblen(cstate->file_encoding, tptr); else tptr++; @@ -1366,7 +1351,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, CopySendChar(cstate, escapec); start = ptr; /* we include char in next run */ } - if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) + if (IS_HIGHBIT_SET(c) && cstate->edata->encoding_embeds_ascii) ptr += pg_encoding_mblen(cstate->file_encoding, ptr); else ptr++; diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 541176e1980..1fd44a132be 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -14,6 +14,7 @@ #ifndef COPY_H #define COPY_H +#include "executor/execdesc.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -87,13 +88,61 @@ typedef struct CopyFormatOptions List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; +typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); +typedef void (*copy_data_dest_cb) (void *data, int len); + +typedef struct CopyToStateData +{ + /* format-specific routines */ + const struct CopyToRoutine *routine; + + struct CopyToExecutionData *edata; + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ + + CopyFormatOptions opts; + + FmgrInfo *out_functions; /* lookup info for output functions */ +} CopyToStateData; + +typedef struct CopyFromStateData +{ + /* format routine */ + const struct CopyFromRoutine *routine; + + struct CopyFromExecutionData *edata; + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + Oid conversion_proc; /* encoding conversion function */ + + CopyFormatOptions opts; + + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ +} CopyFromStateData; + /* These are private in commands/copy[from|to].c */ typedef struct CopyFromStateData *CopyFromState; typedef struct CopyToStateData *CopyToState; -typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); -typedef void (*copy_data_dest_cb) (void *data, int len); - extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index c8b22af22d8..b91b907e8eb 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -56,29 +56,17 @@ typedef enum CopyInsertMethod * This struct contains all the state variables used throughout a COPY FROM * operation. */ -typedef struct CopyFromStateData +typedef struct CopyFromExecutionData { - /* format routine */ - const struct CopyFromRoutine *routine; - /* low-level state data */ CopySource copy_src; /* type of copy source */ FILE *copy_file; /* used if copy_src == COPY_FILE */ StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ EolType eol_type; /* EOL type of input */ - int file_encoding; /* file or remote side's character encoding */ bool need_transcoding; /* file encoding diff from server? */ Oid conversion_proc; /* encoding conversion function */ - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ Node *whereClause; /* WHERE condition (or NULL) */ @@ -96,8 +84,6 @@ typedef struct CopyFromStateData AttrNumber num_defaults; /* count of att that are missing and have * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ ErrorSaveContext *escontext; /* soft error trapped during in_functions * execution */ uint64 num_errors; /* total number of rows which contained soft @@ -164,7 +150,7 @@ typedef struct CopyFromStateData bool input_reached_eof; /* true if we reached EOF */ bool input_reached_error; /* true if a conversion error happened */ /* Shorthand for number of unconsumed bytes available in input_buf */ -#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) +#define INPUT_BUF_BYTES(edata) ((edata)->input_buf_len - (edata)->input_buf_index) /* * raw_buf holds raw input data read from the data source (file or client @@ -178,10 +164,10 @@ typedef struct CopyFromStateData bool raw_reached_eof; /* true if we reached EOF */ /* Shorthand for number of unconsumed bytes available in raw_buf */ -#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) +#define RAW_BUF_BYTES(edata) ((edata)->raw_buf_len - (edata)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; +} CopyFromExecutionData; extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); -- 2.43.5 [application/octet-stream] 0002-Custom-COPY-format.patch (18.9K, ../../CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com/3-0002-Custom-COPY-format.patch) download | inline diff: From 322512cbcf4b2e6c07e6bd8f9d4b2afc2c400193 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Sun, 13 Jul 2025 10:09:57 -0700 Subject: [PATCH 2/2] Custom COPY format. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- src/backend/commands/Makefile | 1 + src/backend/commands/copy.c | 73 +++++++++------ src/backend/commands/copyformat.c | 134 +++++++++++++++++++++++++++ src/backend/commands/copyfrom.c | 26 ++++-- src/backend/commands/copyfromparse.c | 7 +- src/backend/commands/copyto.c | 28 ++++-- src/include/commands/copy.h | 18 +++- src/include/commands/copyapi.h | 5 + 8 files changed, 244 insertions(+), 48 deletions(-) create mode 100644 src/backend/commands/copyformat.c diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index cb2fbdc7c60..30693ffba27 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -24,6 +24,7 @@ OBJS = \ constraint.o \ conversioncmds.o \ copy.o \ + copyformat.o \ copyfrom.o \ copyfromparse.o \ copyto.o \ diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index fae9c41db65..4e482955544 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -514,6 +514,8 @@ ProcessCopyOptions(ParseState *pstate, bool on_error_specified = false; bool log_verbosity_specified = false; bool reject_limit_specified = false; + bool csv_mode = false; + bool binary = false; ListCell *option; /* Support external use for option sanity checking */ @@ -521,6 +523,8 @@ ProcessCopyOptions(ParseState *pstate, opts_out = (CopyFormatOptions *) palloc0(sizeof(CopyFormatOptions)); opts_out->file_encoding = -1; + opts_out->format = COPY_FORMAT_TEXT; + opts_out->format_ext_id = -1; /* Extract options from the statement node tree */ foreach(option, options) @@ -535,16 +539,29 @@ ProcessCopyOptions(ParseState *pstate, errorConflictingDefElem(defel, pstate); format_specified = true; if (strcmp(fmt, "text") == 0) - /* default format */ ; + opts_out->format = COPY_FORMAT_TEXT; else if (strcmp(fmt, "csv") == 0) - opts_out->csv_mode = true; + { + opts_out->format = COPY_FORMAT_CSV; + csv_mode = true; + } else if (strcmp(fmt, "binary") == 0) - opts_out->binary = true; + { + opts_out->format = COPY_FORMAT_BINARY; + binary = true; + } else - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt), - parser_errposition(pstate, defel->location))); + { + opts_out->format_ext_id = GetCopyFormatExtensionId(defel->defname); + + if (opts_out->format_ext_id == -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", fmt), + parser_errposition(pstate, defel->location))); + + opts_out->format = COPY_FORMAT_EXTENSION; + } } else if (strcmp(defel->defname, "freeze") == 0) { @@ -699,31 +716,31 @@ ProcessCopyOptions(ParseState *pstate, * Check for incompatible options (must do these three before inserting * defaults) */ - if (opts_out->binary && opts_out->delim) + if (binary && opts_out->delim) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("cannot specify %s in BINARY mode", "DELIMITER"))); - if (opts_out->binary && opts_out->null_print) + if (binary && opts_out->null_print) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify %s in BINARY mode", "NULL"))); - if (opts_out->binary && opts_out->default_print) + if (binary && opts_out->default_print) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify %s in BINARY mode", "DEFAULT"))); /* Set defaults for omitted options */ if (!opts_out->delim) - opts_out->delim = opts_out->csv_mode ? "," : "\t"; + opts_out->delim = csv_mode ? "," : "\t"; if (!opts_out->null_print) - opts_out->null_print = opts_out->csv_mode ? "" : "\\N"; + opts_out->null_print = csv_mode ? "" : "\\N"; opts_out->null_print_len = strlen(opts_out->null_print); - if (opts_out->csv_mode) + if (csv_mode) { if (!opts_out->quote) opts_out->quote = "\""; @@ -771,7 +788,7 @@ ProcessCopyOptions(ParseState *pstate, * future-proofing. Likewise we disallow all digits though only octal * digits are actually dangerous. */ - if (!opts_out->csv_mode && + if (!csv_mode && strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", opts_out->delim[0]) != NULL) ereport(ERROR, @@ -779,43 +796,43 @@ ProcessCopyOptions(ParseState *pstate, errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim))); /* Check header */ - if (opts_out->binary && opts_out->header_line != COPY_HEADER_FALSE) + if (binary && opts_out->header_line != COPY_HEADER_FALSE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("cannot specify %s in BINARY mode", "HEADER"))); /* Check quote */ - if (!opts_out->csv_mode && opts_out->quote != NULL) + if (!csv_mode && opts_out->quote != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("COPY %s requires CSV mode", "QUOTE"))); - if (opts_out->csv_mode && strlen(opts_out->quote) != 1) + if (csv_mode && strlen(opts_out->quote) != 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("COPY quote must be a single one-byte character"))); - if (opts_out->csv_mode && opts_out->delim[0] == opts_out->quote[0]) + if (csv_mode && opts_out->delim[0] == opts_out->quote[0]) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY delimiter and quote must be different"))); /* Check escape */ - if (!opts_out->csv_mode && opts_out->escape != NULL) + if (!csv_mode && opts_out->escape != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("COPY %s requires CSV mode", "ESCAPE"))); - if (opts_out->csv_mode && strlen(opts_out->escape) != 1) + if (csv_mode && strlen(opts_out->escape) != 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("COPY escape must be a single one-byte character"))); /* Check force_quote */ - if (!opts_out->csv_mode && (opts_out->force_quote || opts_out->force_quote_all)) + if (!csv_mode && (opts_out->force_quote || opts_out->force_quote_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -829,8 +846,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY FROM"))); /* Check force_notnull */ - if (!opts_out->csv_mode && (opts_out->force_notnull != NIL || - opts_out->force_notnull_all)) + if (!csv_mode && (opts_out->force_notnull != NIL || + opts_out->force_notnull_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -845,8 +862,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY TO"))); /* Check force_null */ - if (!opts_out->csv_mode && (opts_out->force_null != NIL || - opts_out->force_null_all)) + if (!csv_mode && (opts_out->force_null != NIL || + opts_out->force_null_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -870,7 +887,7 @@ ProcessCopyOptions(ParseState *pstate, "NULL"))); /* Don't allow the CSV quote char to appear in the null string. */ - if (opts_out->csv_mode && + if (csv_mode && strchr(opts_out->null_print, opts_out->quote[0]) != NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -906,7 +923,7 @@ ProcessCopyOptions(ParseState *pstate, "DEFAULT"))); /* Don't allow the CSV quote char to appear in the default string. */ - if (opts_out->csv_mode && + if (csv_mode && strchr(opts_out->default_print, opts_out->quote[0]) != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -923,7 +940,7 @@ ProcessCopyOptions(ParseState *pstate, errmsg("NULL specification and DEFAULT specification cannot be the same"))); } /* Check on_error */ - if (opts_out->binary && opts_out->on_error != COPY_ON_ERROR_STOP) + if (binary && opts_out->on_error != COPY_ON_ERROR_STOP) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("only ON_ERROR STOP is allowed in BINARY mode"))); diff --git a/src/backend/commands/copyformat.c b/src/backend/commands/copyformat.c new file mode 100644 index 00000000000..6d1c40833e9 --- /dev/null +++ b/src/backend/commands/copyformat.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * copyformat.c + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/copyformat.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/copyapi.h" +#include "utils/memutils.h" + +typedef struct CopyFormatExtensionInfo +{ + const char *format_name; + const CopyToRoutine *to_routine; + const CopyFromRoutine *from_routine; +} CopyFormatExtensionInfo; + +static CopyFormatExtensionInfo * CopyFormatExtensionArray = NULL; +static int CopyFormatExtensionAssigned = 0; +static int CopyFormatExtensionAllocated = 0; + +/* + * Register new COPY routines for the given name. + * + * When name is used as a COPY format name, routine will be used to process + * the COPY request. See CopyFromRoutine and CopyToRoutine how to implement a + * COPY routine. + * + * If name is already registered, an error is raised. + * + * name is assumed to be a constant string or allocated in storage that will + * never be freed. + * + * fmt, from_routine, and to_routine are assumed to be allocated in storage + * that will never be freed. + */ +void +RegisterCustomCopyFormat(const char *fmt, const CopyToRoutine *to_routine, + const CopyFromRoutine *from_routine) +{ + CopyFormatExtensionInfo *info; + + for (int i = 0; i < CopyFormatExtensionAssigned; i++) + { + if (strcmp(fmt, CopyFormatExtensionArray[i].format_name) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("custom COPY fromat \"%s\" already exists", + fmt))); + } + + /* If there is no array yet, create one */ + if (CopyFormatExtensionAllocated == 0) + { + CopyFormatExtensionAllocated = 16; + CopyFormatExtensionArray = (CopyFormatExtensionInfo *) + MemoryContextAlloc(TopMemoryContext, + CopyFormatExtensionAllocated * sizeof(CopyFormatExtensionInfo)); + } + + /* If there's an array but it's currently full, expand it */ + if (CopyFormatExtensionAssigned >= CopyFormatExtensionAllocated) + { + int i = pg_nextpower2_32(CopyFormatExtensionAssigned + 1); + + CopyFormatExtensionArray = + repalloc(CopyFormatExtensionArray, sizeof(CopyFormatExtensionInfo) * i); + CopyFormatExtensionAllocated = i; + } + + /* Register the format */ + info = &CopyFormatExtensionArray[CopyFormatExtensionAssigned++]; + info->format_name = fmt; + info->from_routine = from_routine; + info->to_routine = to_routine; +} + +/* + * Get an integer ID of the given name of an custom COPY format extension. + * + * Within the lifetime of a particular backend, the same name will be + * mapped to the same ID every time. IDs of COPY formats that are not + * loaded during shared_preload_libraries time are not stable across + * backends. + */ +int +GetCopyFormatExtensionId(const char *fmt) +{ + for (int i = 0; i < CopyFormatExtensionAssigned; i++) + { + if (strcmp(CopyFormatExtensionArray[i].format_name, fmt) == 0) + return i; + } + + return -1; +} + +const CopyToRoutine * +GetCustomCopyToRoutine(int extension_id) +{ + if (extension_id >= CopyFormatExtensionAssigned || extension_id < 0) + return NULL; + + if (!CopyFormatExtensionArray[extension_id].to_routine) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("COPY format \"%s\" does not support COPY TO routine", + CopyFormatExtensionArray[extension_id].format_name))); + + return CopyFormatExtensionArray[extension_id].to_routine; +} + +const CopyFromRoutine * +GetCustomCopyFromRoutine(int extension_id) +{ + if (extension_id >= CopyFormatExtensionAssigned || extension_id < 0) + return NULL; + + if (!CopyFormatExtensionArray[extension_id].to_routine) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("COPY format \"%s\" does not support COPY FROM routine", + CopyFormatExtensionArray[extension_id].format_name))); + + return CopyFormatExtensionArray[extension_id].from_routine; +} diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 03779838654..fc38caaca24 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -155,13 +155,25 @@ static const CopyFromRoutine CopyFromRoutineBinary = { static const CopyFromRoutine * CopyFromGetRoutine(const CopyFormatOptions *opts) { - if (opts->csv_mode) - return &CopyFromRoutineCSV; - else if (opts->binary) - return &CopyFromRoutineBinary; + const CopyFromRoutine *routine; - /* default is text */ - return &CopyFromRoutineText; + switch (opts->format) + { + case COPY_FORMAT_CSV: + routine = &CopyFromRoutineCSV; + break; + case COPY_FORMAT_TEXT: + routine = &CopyFromRoutineText; + break; + case COPY_FORMAT_BINARY: + routine = &CopyFromRoutineBinary; + break; + case COPY_FORMAT_EXTENSION: + routine = GetCustomCopyFromRoutine(opts->format_ext_id); + break; + } + + return routine; } /* Implementation of the start callback for text and CSV formats */ @@ -263,7 +275,7 @@ CopyFromErrorCallback(void *arg) edata->cur_relname); return; } - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* can't usefully display the data */ if (edata->cur_attname) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 595dc84b172..0b94d8851f3 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -171,7 +171,7 @@ ReceiveCopyBegin(CopyFromState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0); int i; pq_beginmessage(&buf, PqMsg_CopyInResponse); @@ -754,7 +754,7 @@ bool NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) { return NextCopyFromRawFieldsInternal(cstate, fields, nfields, - cstate->opts.csv_mode); + cstate->opts.format == COPY_FORMAT_CSV); } /* @@ -782,7 +782,8 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields bool done = false; /* only available for text or csv input */ - Assert(!cstate->opts.binary); + Assert(cstate->opts.format == COPY_FORMAT_TEXT || + cstate->opts.format == COPY_FORMAT_CSV); /* on input check that the header line is correct if needed */ if (edata->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index e90fab6d958..842c5efa8d5 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -160,13 +160,25 @@ static const CopyToRoutine CopyToRoutineBinary = { static const CopyToRoutine * CopyToGetRoutine(const CopyFormatOptions *opts) { - if (opts->csv_mode) - return &CopyToRoutineCSV; - else if (opts->binary) - return &CopyToRoutineBinary; + const CopyToRoutine *routine; - /* default is text */ - return &CopyToRoutineText; + switch (opts->format) + { + case COPY_FORMAT_CSV: + routine = &CopyToRoutineCSV; + break; + case COPY_FORMAT_TEXT: + routine = &CopyToRoutineText; + break; + case COPY_FORMAT_BINARY: + routine = &CopyToRoutineBinary; + break; + case COPY_FORMAT_EXTENSION: + routine = GetCustomCopyToRoutine(opts->format_ext_id); + break; + } + + return routine; } /* Implementation of the start callback for text and CSV formats */ @@ -199,7 +211,7 @@ CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc) colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) CopyAttributeOutCSV(cstate, colname, false); else CopyAttributeOutText(cstate, colname); @@ -376,7 +388,7 @@ SendCopyBegin(CopyToState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0); int i; pq_beginmessage(&buf, PqMsg_CopyOutResponse); diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 1fd44a132be..73660e694ce 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -49,6 +49,17 @@ typedef enum CopyLogVerbosityChoice COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */ } CopyLogVerbosityChoice; +/* + * Represents the file format processed by COPY command. + */ +typedef enum CopyFormatType +{ + COPY_FORMAT_CSV = 0, + COPY_FORMAT_TEXT, + COPY_FORMAT_BINARY, + COPY_FORMAT_EXTENSION, +} CopyFormatType; + /* * A struct to hold COPY options, in a parsed form. All of these are related * to formatting, except for 'freeze', which doesn't really belong here, but @@ -59,9 +70,10 @@ typedef struct CopyFormatOptions /* parameters from the COPY command */ int file_encoding; /* file or remote side's character encoding, * -1 if not specified */ - bool binary; /* binary format? */ + CopyFormatType format; + int format_ext_id; /* custom COPY format id, -1 if format is + * built-in format */ bool freeze; /* freeze rows on loading? */ - bool csv_mode; /* Comma Separated Value format? */ int header_line; /* number of lines to skip or COPY_HEADER_XXX * value (see the above) */ char *null_print; /* NULL marker string (server encoding!) */ @@ -163,6 +175,8 @@ extern uint64 CopyFrom(CopyFromState cstate); extern DestReceiver *CreateCopyDestReceiver(void); +extern int GetCopyFormatExtensionId(const char *fmt); + /* * internal prototypes */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 2a2d2f9876b..b9e29dddd5b 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -102,4 +102,9 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern void RegisterCustomCopyFormat(const char *fmt, const CopyToRoutine *to_routine, + const CopyFromRoutine *from_routine); +extern const CopyToRoutine *GetCustomCopyToRoutine(int extension_id); +extern const CopyFromRoutine *GetCustomCopyFromRoutine(int extension_id); + #endif /* COPYAPI_H */ -- 2.43.5 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-14 08:38 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-07-14 08:38 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoB0Z3gkOGALK3pXYmGTWATVvgDAmn-yXGp2mX64S-YrSw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 03:28:16 +0900, Masahiko Sawada <[email protected]> wrote: > I've reviewed the 0001 and 0002 patches. The API implemented in the > 0002 patch looks good to me, but I'm concerned about the capsulation > of copy state data. With the v42 patches, we pass the whole > CopyToStateData to the extension codes, but most of the fields in > CopyToStateData are internal working state data that shouldn't be > exposed to extensions. I think we need to sort out which fields are > exposed or not. That way, it would be safer and we would be able to > avoid exposing copyto_internal.h and extensions would not need to > include copyfrom_internal.h. FYI: We discussed this so far. For example: https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m... > I think we can move CopyToState to copy.h and we don't > need to have set/get functions for its fields. https://www.postgresql.org/message-id/flat/CAD21AoBpWFU4k-_bwrTq0AkFSAdwQqhAsSW188STmu9HxLJ0nQ%40mai... > > What does "private" mean here? I thought that it means that > > "PostgreSQL itself can use it". But it seems that you mean > > that "PostgreSQL itself and custom format extensions can use > > it but other extensions can't use it". > > > > I'm not familiar with "_internal.h" in PostgreSQL but is > > "_internal.h" for the latter "private" mean? > > My understanding is that we don't strictly prohibit _internal.h from > being included in out of core files. For example, file_fdw.c includes > copyfrom_internal.h in order to access some fields of CopyFromState. In general, I agree that we should export only needed information. How about adding accessors instead of splitting Copy{From,To}State to Copy{From,To}ExecutionData? If we use the accessors approach, we can export only needed information step by step without breaking ABI. The built-in formats can keep using Copy{From,To}State directly with the accessors approach. We can avoid any performance regression of the built-in formats. If we split Copy{From,To}State to Copy{From,To}ExecutionData, performance may be changed. > I've implemented a draft patch for that idea. In the 0001 patch, I > moved fields that are related to internal working state from > CopyToStateData to CopyToExectuionData. COPY routine APIs pass a > pointer of CopyToStateData but extensions can access only fields > except for CopyToExectuionData. In the 0002 patch, I've implemented > the registration API and some related APIs based on your v42 patch. > I've made similar changes to COPY FROM codes too. > > The patch is a very PoC phase and we would need to scrutinize the > fields that should or should not be exposed. Feedback is very welcome. Based on our sample extensions [1][2], the following fields may be minimal. I added "(*)" marks that exist in Copy{From,To}StateDate in your patch. Other fields exist in Copy{From,To}ExecutionData. We need to export them to extensions. We can hide fields in Copy{From,To}StateData not listed here. FROM: - attnumlist (*) - bytes_processed - cur_attname - escontext - in_functions (*) - input_buf - input_reached_eof - line_buf - opts (*) - raw_buf - raw_buf_index - raw_buf_len - rel (*) - typioparams (*) TO: - attnumlist (*) - fe_msgbuf - opts (*) [1] https://github.com/kou/pg-copy-arrow/ [2] https://github.com/MasahikoSawada/pg_copy_jsonlines/ Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-15 07:54 Sutou Kouhei <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-07-15 07:54 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 17:38:03 +0900 (JST), Sutou Kouhei <[email protected]> wrote: >> I've reviewed the 0001 and 0002 patches. The API implemented in the >> 0002 patch looks good to me, but I'm concerned about the capsulation >> of copy state data. With the v42 patches, we pass the whole >> CopyToStateData to the extension codes, but most of the fields in >> CopyToStateData are internal working state data that shouldn't be >> exposed to extensions. I think we need to sort out which fields are >> exposed or not. That way, it would be safer and we would be able to >> avoid exposing copyto_internal.h and extensions would not need to >> include copyfrom_internal.h. > In general, I agree that we should export only needed > information. > > How about adding accessors instead of splitting > Copy{From,To}State to Copy{From,To}ExecutionData? If we use > the accessors approach, we can export only needed > information step by step without breaking ABI. Another idea: We'll add Copy{From,To}State::opaque eventually. (For example, the v40-0003 patch includes it.) How about using it to hide fields only for built-in formats? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-15 12:36 Andres Freund <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Andres Freund @ 2025-07-15 12:36 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, On 2025-07-14 03:28:16 +0900, Masahiko Sawada wrote: > I've reviewed the 0001 and 0002 patches. The API implemented in the > 0002 patch looks good to me, but I'm concerned about the capsulation > of copy state data. With the v42 patches, we pass the whole > CopyToStateData to the extension codes, but most of the fields in > CopyToStateData are internal working state data that shouldn't be > exposed to extensions. I think we need to sort out which fields are > exposed or not. That way, it would be safer and we would be able to > avoid exposing copyto_internal.h and extensions would not need to > include copyfrom_internal.h. > > I've implemented a draft patch for that idea. In the 0001 patch, I > moved fields that are related to internal working state from > CopyToStateData to CopyToExectuionData. COPY routine APIs pass a > pointer of CopyToStateData but extensions can access only fields > except for CopyToExectuionData. In the 0002 patch, I've implemented > the registration API and some related APIs based on your v42 patch. > I've made similar changes to COPY FROM codes too. I've not followed the development of this patch - but I continue to be concerned about the performance impact it has as-is and the amount of COPY performance improvements it forecloses. This seems to add yet another layer of indirection to a lot of hot functions like CopyGetData() etc. Greetings, Andres Freund ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-17 20:33 Masahiko Sawada <[email protected]> parent: Andres Freund <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-07-17 20:33 UTC (permalink / raw) To: Andres Freund <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jul 15, 2025 at 5:37 AM Andres Freund <[email protected]> wrote: > > Hi, > > On 2025-07-14 03:28:16 +0900, Masahiko Sawada wrote: > > I've reviewed the 0001 and 0002 patches. The API implemented in the > > 0002 patch looks good to me, but I'm concerned about the capsulation > > of copy state data. With the v42 patches, we pass the whole > > CopyToStateData to the extension codes, but most of the fields in > > CopyToStateData are internal working state data that shouldn't be > > exposed to extensions. I think we need to sort out which fields are > > exposed or not. That way, it would be safer and we would be able to > > avoid exposing copyto_internal.h and extensions would not need to > > include copyfrom_internal.h. > > > > I've implemented a draft patch for that idea. In the 0001 patch, I > > moved fields that are related to internal working state from > > CopyToStateData to CopyToExectuionData. COPY routine APIs pass a > > pointer of CopyToStateData but extensions can access only fields > > except for CopyToExectuionData. In the 0002 patch, I've implemented > > the registration API and some related APIs based on your v42 patch. > > I've made similar changes to COPY FROM codes too. > > I've not followed the development of this patch - but I continue to be > concerned about the performance impact it has as-is and the amount of COPY > performance improvements it forecloses. > > This seems to add yet another layer of indirection to a lot of hot functions > like CopyGetData() etc. > The most refactoring works have been done by commit 7717f6300 and 2e4127b6d with a slight performance gain. At this stage, we're trying to introduce the registration API so that extensions can provide their callbacks to the core. Some functions required for I/O such as CopyGetData() and CopySendEndOfRow() would be exposed but I'm not going to add additional indirection function call layers. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-17 20:44 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-07-17 20:44 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jul 15, 2025 at 12:54 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 14 Jul 2025 17:38:03 +0900 (JST), > Sutou Kouhei <[email protected]> wrote: > > >> I've reviewed the 0001 and 0002 patches. The API implemented in the > >> 0002 patch looks good to me, but I'm concerned about the capsulation > >> of copy state data. With the v42 patches, we pass the whole > >> CopyToStateData to the extension codes, but most of the fields in > >> CopyToStateData are internal working state data that shouldn't be > >> exposed to extensions. I think we need to sort out which fields are > >> exposed or not. That way, it would be safer and we would be able to > >> avoid exposing copyto_internal.h and extensions would not need to > >> include copyfrom_internal.h. > > > In general, I agree that we should export only needed > > information. > > > > How about adding accessors instead of splitting > > Copy{From,To}State to Copy{From,To}ExecutionData? If we use > > the accessors approach, we can export only needed > > information step by step without breaking ABI. Yeah, while it can export required fields without breaking ABI, I'm concerned that setter and getter functions could be bloated if we need to have them for many fields. > > Another idea: We'll add Copy{From,To}State::opaque > eventually. (For example, the v40-0003 patch includes it.) > > How about using it to hide fields only for built-in formats? What is the difference between your idea and splitting CopyToState into CopyToState and CopyToExecutionData? Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-18 09:49 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-07-18 09:49 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoAQkjU=o0nX4y0jtX0BnsrqA04g2ABqrUwjT88YeEWarA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:33:13 -0700, Masahiko Sawada <[email protected]> wrote: >> I've not followed the development of this patch - but I continue to be >> concerned about the performance impact it has as-is and the amount of COPY >> performance improvements it forecloses. >> >> This seems to add yet another layer of indirection to a lot of hot functions >> like CopyGetData() etc. >> > > The most refactoring works have been done by commit 7717f6300 and > 2e4127b6d with a slight performance gain. At this stage, we're trying > to introduce the registration API so that extensions can provide their > callbacks to the core. Some functions required for I/O such as > CopyGetData() and CopySendEndOfRow() would be exposed but I'm not > going to add additional indirection function call layers. I think Andres is talking about any indirection not only indirection function call. In this case, "cstate->XXX" -> "cstate->edata->XXX". It's also mentioned in my e-mail. I'm not sure whether it has performance impact but it's better that we benchmark to confirm whether there is any performance impact or not with the Copy{From,To}ExecutionData approach. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-18 10:05 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-07-18 10:05 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700, Masahiko Sawada <[email protected]> wrote: >> > How about adding accessors instead of splitting >> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use >> > the accessors approach, we can export only needed >> > information step by step without breaking ABI. > > Yeah, while it can export required fields without breaking ABI, I'm > concerned that setter and getter functions could be bloated if we need > to have them for many fields. In general, I choose this approach in my projects even when I need to define many accessors. Because I can hide implementation details from users. I can change implementation details without breaking API/ABI. But PostgreSQL isn't my project. Is there any guideline for PostgreSQL API(/ABI?) design that we can refer for this case? FYI: We need to export at least the following fields: https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7... > FROM: > > - attnumlist (*) > - bytes_processed > - cur_attname > - escontext > - in_functions (*) > - input_buf > - input_reached_eof > - line_buf > - opts (*) > - raw_buf > - raw_buf_index > - raw_buf_len > - rel (*) > - typioparams (*) > > TO: > > - attnumlist (*) > - fe_msgbuf > - opts (*) Here are pros/cons of the Copy{From,To}ExecutionData approach, right? Pros: 1. We can hide internal data from extensions Cons: 1. Built-in format routines need to refer fields via Copy{From,To}ExecutionData. * This MAY has performance impact. If there is no performance impact, this is not a cons. 2. API/ABI compatibility will be broken when we change exported fields. * I'm not sure whether this is a cons in the PostgreSQL design. Here are pros/cons of the accessors approach: Pros: 1. We can hide internal data from extensions 2. We can export new fields change field names without breaking API/ABI compatibility 3. We don't need to change built-in format routines. So we can assume that there is no performance impact. Cons: 1. We may need to define many accessors * I'm not sure whether this is a cons in the PostgreSQL design. >> Another idea: We'll add Copy{From,To}State::opaque >> eventually. (For example, the v40-0003 patch includes it.) >> >> How about using it to hide fields only for built-in formats? > > What is the difference between your idea and splitting CopyToState > into CopyToState and CopyToExecutionData? 1. We don't need to manage 2 similar data for built-in formats and extensions. * Build-in formats use CopyToExecutionData and extensions use opaque. 2. We can introduce registration API now. * We can work on this topic AFTER we introduce registration API. * e.g.: Add registration API -> Add opaque -> Use opaque for internal fields (we will benchmark this implementation at this time) Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-28 19:33 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-07-28 19:33 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jul 18, 2025 at 3:05 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> > How about adding accessors instead of splitting > >> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use > >> > the accessors approach, we can export only needed > >> > information step by step without breaking ABI. > > > > Yeah, while it can export required fields without breaking ABI, I'm > > concerned that setter and getter functions could be bloated if we need > > to have them for many fields. > > In general, I choose this approach in my projects even when > I need to define many accessors. Because I can hide > implementation details from users. I can change > implementation details without breaking API/ABI. > > But PostgreSQL isn't my project. Is there any guideline for > PostgreSQL API(/ABI?) design that we can refer for this > case? As far as I know there is no official guideline for PostgreSQL API and ABI design, but I've never seen structs having more than 10 getter and setter in PostgreSQL source code. > > FYI: We need to export at least the following fields: > > https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7... > > > FROM: > > > > - attnumlist (*) > > - bytes_processed > > - cur_attname > > - escontext > > - in_functions (*) > > - input_buf > > - input_reached_eof > > - line_buf > > - opts (*) > > - raw_buf > > - raw_buf_index > > - raw_buf_len > > - rel (*) > > - typioparams (*) > > > > TO: > > > > - attnumlist (*) > > - fe_msgbuf > > - opts (*) I think we can think of the minimum list of fields that we need to expose. For instance, fields used for buffered reads for COPY FROM such as input_buf and raw_buf related fields don't necessarily need to be exposed as extension can implement it in its own way. We can start with the supporting simple copy format extensions like that read and parse the binary data from the data source and fill 'values' and 'nulls' arrays as output. Considering these facts, it might be sufficient for copy format extensions if they could access 'rel', 'attnumlist', and 'opts' in both COPY FROM and COPY TO (and CopyFromErrorCallback related fields for COPY FROM). Apart from this, we might want to reorganize CopyFromStateData fields and CopyToStateData fields since they have mixed fields of general purpose fields for COPY operations (e.g., num_defaults, whereClause, and range_table) and built-in format specific fields (e.g., line_buf and input_buf). Text and CSV formats are using some fields for parsing fields with buffered reads so one idea is that we move related fields to another struct so that both built-in formats (text and CSV) and external extensions that want to use the buffered reads for text parsing can use this functionality. > Here are pros/cons of the Copy{From,To}ExecutionData > approach, right? > > Pros: > 1. We can hide internal data from extensions > > Cons: > 1. Built-in format routines need to refer fields via > Copy{From,To}ExecutionData. > * This MAY has performance impact. If there is no > performance impact, this is not a cons. > 2. API/ABI compatibility will be broken when we change > exported fields. > * I'm not sure whether this is a cons in the PostgreSQL > design. > > Here are pros/cons of the accessors approach: > > Pros: > 1. We can hide internal data from extensions > 2. We can export new fields change field names > without breaking API/ABI compatibility > 3. We don't need to change built-in format routines. > So we can assume that there is no performance impact. > > Cons: > 1. We may need to define many accessors > * I'm not sure whether this is a cons in the PostgreSQL > design. I agree with the summary. > >> Another idea: We'll add Copy{From,To}State::opaque > >> eventually. (For example, the v40-0003 patch includes it.) > >> > >> How about using it to hide fields only for built-in formats? > > > > What is the difference between your idea and splitting CopyToState > > into CopyToState and CopyToExecutionData? > > 1. We don't need to manage 2 similar data for built-in > formats and extensions. > * Build-in formats use CopyToExecutionData and extensions > use opaque. > 2. We can introduce registration API now. > * We can work on this topic AFTER we introduce > registration API. > * e.g.: Add registration API -> Add opaque -> Use opaque > for internal fields (we will benchmark this > implementation at this time) What if we find performance overhead in built-in format cases after introducing opaque data? I personally would like to avoid merging the registration API (i.e., supporting custom copy formats) while being unsure about the overall design ahead and the potential performance impact by following patches. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-07-29 05:19 Masahiko Sawada <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-07-29 05:19 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jul 28, 2025 at 12:33 PM Masahiko Sawada <[email protected]> wrote: > > On Fri, Jul 18, 2025 at 3:05 AM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAD21AoAZL2RzPM4RLOJKm_73z5LXq2_VOVF+S+T0tnbjHdWTFA@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 17 Jul 2025 13:44:11 -0700, > > Masahiko Sawada <[email protected]> wrote: > > > > >> > How about adding accessors instead of splitting > > >> > Copy{From,To}State to Copy{From,To}ExecutionData? If we use > > >> > the accessors approach, we can export only needed > > >> > information step by step without breaking ABI. > > > > > > Yeah, while it can export required fields without breaking ABI, I'm > > > concerned that setter and getter functions could be bloated if we need > > > to have them for many fields. > > > > In general, I choose this approach in my projects even when > > I need to define many accessors. Because I can hide > > implementation details from users. I can change > > implementation details without breaking API/ABI. > > > > But PostgreSQL isn't my project. Is there any guideline for > > PostgreSQL API(/ABI?) design that we can refer for this > > case? > > As far as I know there is no official guideline for PostgreSQL API and > ABI design, but I've never seen structs having more than 10 getter and > setter in PostgreSQL source code. > > > > > FYI: We need to export at least the following fields: > > > > https://www.postgresql.org/message-id/flat/20250714.173803.865595983884510428.kou%40clear-code.com#7... > > > > > FROM: > > > > > > - attnumlist (*) > > > - bytes_processed > > > - cur_attname > > > - escontext > > > - in_functions (*) > > > - input_buf > > > - input_reached_eof > > > - line_buf > > > - opts (*) > > > - raw_buf > > > - raw_buf_index > > > - raw_buf_len > > > - rel (*) > > > - typioparams (*) > > > > > > TO: > > > > > > - attnumlist (*) > > > - fe_msgbuf > > > - opts (*) > > I think we can think of the minimum list of fields that we need to > expose. For instance, fields used for buffered reads for COPY FROM > such as input_buf and raw_buf related fields don't necessarily need to > be exposed as extension can implement it in its own way. We can start > with the supporting simple copy format extensions like that read and > parse the binary data from the data source and fill 'values' and > 'nulls' arrays as output. Considering these facts, it might be > sufficient for copy format extensions if they could access 'rel', > 'attnumlist', and 'opts' in both COPY FROM and COPY TO (and > CopyFromErrorCallback related fields for COPY FROM). > > Apart from this, we might want to reorganize CopyFromStateData fields > and CopyToStateData fields since they have mixed fields of general > purpose fields for COPY operations (e.g., num_defaults, whereClause, > and range_table) and built-in format specific fields (e.g., line_buf > and input_buf). Text and CSV formats are using some fields for parsing > fields with buffered reads so one idea is that we move related fields > to another struct so that both built-in formats (text and CSV) and > external extensions that want to use the buffered reads for text > parsing can use this functionality. So probably it might be worth refactoring the codes in terms of: 1. hiding internal data from format callbacks 2. separating format-specific fields from the main state data. I categorized the fields in CopyFromStateData. I think there are roughly three different kind of fields mixed there: 1. fields used only the core (not by format callback) - filename - is_program - whereClause - cur_relname - copycontext - defmap - num_defaults - volatile_defexprs - range_table - rtrperminfos - qualexpr - transition_capture 2. fields used by both the core and format callbacks - rel - attnumlist - cur_lineno - cur_attname - cur_attval - relname_only - num_errors - opts - in_functions - typioparams - escontext - defexprs - Input-related fields - copy_src - copy_file - fe_msgbuf - data_source_cb - byteprocessed 3. built-in format specific fields (mostly for text and csv) - eol_type - defaults - Encoding related fields - file_encoding - need_transcoding - conversion_proc - convert_select_flags - raw data pointers - max_fields - raw_fields - attribute_buf - line_buf related fields - line_buf - line_buf_valid - input_buf related fields - input_buf - input_buf_index - input_buf_len - input_reached_eof - input_reached_error - raw_buf related fields - raw_buf - raw_buf_index - raw_buf_len - raw_reached_eof The fields in 1 are mostly static fields, and the fields in 2 and 3 are likely to be accessed in hot functions during COPY FROM. Would it be a good idea to restructure these fields so that we can hide the fields in 1 from callback functions and having the fields in 3 in a separate format-specific struct that can be accessed via an opaque pointer? But could the latter change potentially cause performance overheads? Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-08-14 06:36 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-08-14 06:36 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC+gf=3w7XiA4LQnvx0g@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 28 Jul 2025 22:19:36 -0700, Masahiko Sawada <[email protected]> wrote: > The fields in 1 are mostly static fields, and the fields in 2 and 3 > are likely to be accessed in hot functions during COPY FROM. Would it > be a good idea to restructure these fields so that we can hide the > fields in 1 from callback functions and having the fields in 3 in a > separate format-specific struct that can be accessed via an opaque > pointer? But could the latter change potentially cause performance > overheads? Yes. It changes memory layout (1 continuous memory chunk -> 2 separated memory chunks) and introduces indirect member accesses (x->y -> x->z->y). They may not have performance impact but we need to measure it if we want to use this approach. BTW, how about the following approach? copyapi.h: typedef struct CopyToStateData { /* public members */ /* ... */ } CopyToStateData; copyto.c: typedef struct CopyToStateInternalData { CopyToStateData parent; /* private members */ /* ... */ } CopyToStateInternalData; We export CopyToStateData only with public members. We don't export CopyToStateInternalData that has members only for built-in formats. CopyToStateInternalData has the same memory layout as CopyToStateData. So we can use CopyToStateInternalData as CopyToStateData. We use CopyToStateData not CopyToStateInternalData in public API. We cast CopyToStateData to CopyToStateInternalData when we need to use private members: static void CopySendData(CopyToState cstate, const void *databuf, int datasize) { CopyToStateInternal cstate_internal = (CopyToStateInternal) cstate; appendBinaryStringInfo(cstate_internal->fe_msgbuf, databuf, datasize); } It's still direct member access. With this approach, we can keep the same memory layout (1 continuous memory chunk) and direct member access. I think that this approach doesn't have performance impact. See the attached patch for PoC of this approach. Drawback: This approach always allocates CopyToStateInternalData not CopyToStateData. So we need to allocate needless memories for extensions. But this will prevent performance regression of built-in formats. Is it acceptable? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-08 21:08 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-09-08 21:08 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Aug 13, 2025 at 11:37 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC+gf=3w7XiA4LQnvx0g@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 28 Jul 2025 22:19:36 -0700, > Masahiko Sawada <[email protected]> wrote: > > > The fields in 1 are mostly static fields, and the fields in 2 and 3 > > are likely to be accessed in hot functions during COPY FROM. Would it > > be a good idea to restructure these fields so that we can hide the > > fields in 1 from callback functions and having the fields in 3 in a > > separate format-specific struct that can be accessed via an opaque > > pointer? But could the latter change potentially cause performance > > overheads? > > Yes. It changes memory layout (1 continuous memory chunk -> > 2 separated memory chunks) and introduces indirect member > accesses (x->y -> x->z->y). I think fields accessed by the hot functions are limited. If we assemble fields required by hot functions into one struct and pass it to them can we deal with the latter? For example, we assemble the fields in 3 I mentioned above (i.e., built-in format specific fields) into say CopyFromStateBuiltin and pass it to CopyReadLine() function and the following functions, instead of CopyFromState. Since there are some places where we need to access to CopyFromState (e.g., CopyGetData()), CopyFromStateBuiltin needs to have a pointer to CopyFromState as well. > They may not have performance > impact but we need to measure it if we want to use this > approach. Agreed. > BTW, how about the following approach? > > copyapi.h: > > typedef struct CopyToStateData > { > /* public members */ > /* ... */ > } CopyToStateData; > > copyto.c: > > typedef struct CopyToStateInternalData > { > CopyToStateData parent; > > /* private members */ > /* ... */ > } CopyToStateInternalData; > > We export CopyToStateData only with public members. We don't > export CopyToStateInternalData that has members only for > built-in formats. > > CopyToStateInternalData has the same memory layout as > CopyToStateData. So we can use CopyToStateInternalData as > CopyToStateData. > > We use CopyToStateData not CopyToStateInternalData in public > API. We cast CopyToStateData to CopyToStateInternalData when > we need to use private members: > > static void > CopySendData(CopyToState cstate, const void *databuf, int datasize) > { > CopyToStateInternal cstate_internal = (CopyToStateInternal) cstate; > appendBinaryStringInfo(cstate_internal->fe_msgbuf, databuf, datasize); > } > > It's still direct member access. > > > With this approach, we can keep the same memory layout (1 > continuous memory chunk) and direct member access. I think > that this approach doesn't have performance impact. > > See the attached patch for PoC of this approach. > > Drawback: This approach always allocates > CopyToStateInternalData not CopyToStateData. So we need to > allocate needless memories for extensions. But this will > prevent performance regression of built-in formats. Is it > acceptable? While this approach could make sense to avoid potential performance overheads for built-in format, I find it's somewhat odd that extensions cannot allocate memory for its working state having CopyToStateData as the base type. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-09 02:50 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-09-09 02:50 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoCCjKA77xkUxx59qJ8an_G_58Mry_EtCEcFgd=g9N2xew@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 8 Sep 2025 14:08:16 -0700, Masahiko Sawada <[email protected]> wrote: >> > The fields in 1 are mostly static fields, and the fields in 2 and 3 >> > are likely to be accessed in hot functions during COPY FROM. Would it >> > be a good idea to restructure these fields so that we can hide the >> > fields in 1 from callback functions and having the fields in 3 in a >> > separate format-specific struct that can be accessed via an opaque >> > pointer? But could the latter change potentially cause performance >> > overheads? >> >> Yes. It changes memory layout (1 continuous memory chunk -> >> 2 separated memory chunks) and introduces indirect member >> accesses (x->y -> x->z->y). > > I think fields accessed by the hot functions are limited. If we > assemble fields required by hot functions into one struct and pass it > to them can we deal with the latter? For example, we assemble the > fields in 3 I mentioned above (i.e., built-in format specific fields) > into say CopyFromStateBuiltin and pass it to CopyReadLine() function > and the following functions, instead of CopyFromState. Since there are > some places where we need to access to CopyFromState (e.g., > CopyGetData()), CopyFromStateBuiltin needs to have a pointer to > CopyFromState as well. It can change indirect member accesses (built-in format specific members can be direct access but other members in CopyFromState are indirect access) but it doesn't change 2 separated memory chunks. If this approach has performance impact and it's caused by indirect member accesses for built-in format specific members, your suggestion will work. If performance impact is caused by another reason, your suggestion may not work. Anyway, we need to measure performance to proceed with this approach. If we can confirm that this approach doesn't have any performance impact, we can use the original your idea. Do you have any idea how to measure performance of this approach? We did it when we introduce Copy{To,From}Routine. But it was difficult to evaluate the results: * I don't have machines for stable benchmark results * We may not be able to use them for the final decision * Most results showed performance improvement but there was a result showed mysterious result[1] [1] https://www.postgresql.org/message-id/flat/CAEG8a3LUBcvjwqgt6AijJmg67YN_b_NZ4Kzoxc_dH4rpAq0pKg%40mai... >> Drawback: This approach always allocates >> CopyToStateInternalData not CopyToStateData. So we need to >> allocate needless memories for extensions. But this will >> prevent performance regression of built-in formats. Is it >> acceptable? > > While this approach could make sense to avoid potential performance > overheads for built-in format, I find it's somewhat odd that > extensions cannot allocate memory for its working state having > CopyToStateData as the base type. Is it important? We'll provide a opaque member for extensions. Extensions should use the opaque member instead of extending Copy{From,To}StateData. I don't object your approach but we need a good way to measure performance. If we use this approach, we can omit it for now and we can revisit your approach later without breaking compatibility. How about using this approach if we can't find a good way to measure performance? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-09 20:15 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-09-09 20:15 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Sep 8, 2025 at 7:50 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoCCjKA77xkUxx59qJ8an_G_58Mry_EtCEcFgd=g9N2xew@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 8 Sep 2025 14:08:16 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> > The fields in 1 are mostly static fields, and the fields in 2 and 3 > >> > are likely to be accessed in hot functions during COPY FROM. Would it > >> > be a good idea to restructure these fields so that we can hide the > >> > fields in 1 from callback functions and having the fields in 3 in a > >> > separate format-specific struct that can be accessed via an opaque > >> > pointer? But could the latter change potentially cause performance > >> > overheads? > >> > >> Yes. It changes memory layout (1 continuous memory chunk -> > >> 2 separated memory chunks) and introduces indirect member > >> accesses (x->y -> x->z->y). > > > > I think fields accessed by the hot functions are limited. If we > > assemble fields required by hot functions into one struct and pass it > > to them can we deal with the latter? For example, we assemble the > > fields in 3 I mentioned above (i.e., built-in format specific fields) > > into say CopyFromStateBuiltin and pass it to CopyReadLine() function > > and the following functions, instead of CopyFromState. Since there are > > some places where we need to access to CopyFromState (e.g., > > CopyGetData()), CopyFromStateBuiltin needs to have a pointer to > > CopyFromState as well. > > It can change indirect member accesses (built-in format > specific members can be direct access but other members in > CopyFromState are indirect access) but it doesn't change 2 > separated memory chunks. Right. IIUC the latter point is related to cache locality. But I'm not sure how much the latter point affects the performance as currently we don't declare fields to CopyFromState while carefully considering the cache locality even today. Separating a single memory into multiple chunks could even have a positive effect on it. > > If this approach has performance impact and it's caused by > indirect member accesses for built-in format specific > members, your suggestion will work. If performance impact is > caused by another reason, your suggestion may not work. > > Anyway, we need to measure performance to proceed with this > approach. If we can confirm that this approach doesn't have > any performance impact, we can use the original your idea. > > Do you have any idea how to measure performance of this > approach? I think we can start with measuring the entire COPY execution time with several scenarios as we did previously. For example, reading a huge file with a single column value and with many columns etc. > > We did it when we introduce Copy{To,From}Routine. But it was > difficult to evaluate the results: > > * I don't have machines for stable benchmark results > * We may not be able to use them for the final decision > * Most results showed performance improvement but > there was a result showed mysterious result[1] Perhaps measuring cache-misses help to see how changes could affect the performance? > > [1] https://www.postgresql.org/message-id/flat/CAEG8a3LUBcvjwqgt6AijJmg67YN_b_NZ4Kzoxc_dH4rpAq0pKg%40mai... > > >> Drawback: This approach always allocates > >> CopyToStateInternalData not CopyToStateData. So we need to > >> allocate needless memories for extensions. But this will > >> prevent performance regression of built-in formats. Is it > >> acceptable? > > > > While this approach could make sense to avoid potential performance > > overheads for built-in format, I find it's somewhat odd that > > extensions cannot allocate memory for its working state having > > CopyToStateData as the base type. > > Is it important? We'll provide a opaque member for > extensions. Extensions should use the opaque member instead > of extending Copy{From,To}StateData. I think yes, because it could be a blocker for future improvements that might require a large field to CopyFrom/ToStateData. > I don't object your approach but we need a good way to > measure performance. If we use this approach, we can omit it > for now and we can revisit your approach later without > breaking compatibility. How about using this approach if we > can't find a good way to measure performance? I think it would be better to hear more opinions about this idea and then make a decision, rather than basing our decision on whether or not we can measure its performance, so we can be more confident in the idea we have chosen. While this idea has the above downside, it could make sense because we always allocate the entire CopyFrom/ToStateData even today in spite of some fields being not used at all in binary format and it requires less implementation costs to hide the for-core-only fields. On the other hand, another possible idea is that we have three different structs for categories 1 (core-only), 2 (core and extensions), and 3 (extension-only), and expose only 2 that has a void pointer to 3. The core can allocate the memory for 1 that embeds 2 at the beginning of the fields. While this design looks cleaner and we can minimize overheads due to indirect references, it would require more implementation costs. Which method we choose, I think we need performance measurements in several scenarios to check if performance regressions don't happen unexpectedly. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-10 02:40 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-09-10 02:40 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoCidyfKcpf9-f2Np8kWgkM09c4TjnS1h1hcO_-CCbjeqw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 9 Sep 2025 13:15:43 -0700, Masahiko Sawada <[email protected]> wrote: >> I don't object your approach but we need a good way to >> measure performance. If we use this approach, we can omit it >> for now and we can revisit your approach later without >> breaking compatibility. How about using this approach if we >> can't find a good way to measure performance? > > I think it would be better to hear more opinions about this idea and > then make a decision, rather than basing our decision on whether or > not we can measure its performance, so we can be more confident in the > idea we have chosen. While this idea has the above downside, it could > make sense because we always allocate the entire CopyFrom/ToStateData > even today in spite of some fields being not used at all in binary > format and it requires less implementation costs to hide the > for-core-only fields. On the other hand, another possible idea is that > we have three different structs for categories 1 (core-only), 2 (core > and extensions), and 3 (extension-only), and expose only 2 that has a > void pointer to 3. The core can allocate the memory for 1 that embeds > 2 at the beginning of the fields. While this design looks cleaner and > we can minimize overheads due to indirect references, it would require > more implementation costs. Which method we choose, I think we need > performance measurements in several scenarios to check if performance > regressions don't happen unexpectedly. OK. So the next step is collecting more opinions, right? Could you add key people in this area to Cc to hear their opinions? I'm not familiar with key people in the PostgreSQL community... Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-10 07:36 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-09-10 07:36 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Sep 9, 2025 at 7:41 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoCidyfKcpf9-f2Np8kWgkM09c4TjnS1h1hcO_-CCbjeqw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 9 Sep 2025 13:15:43 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> I don't object your approach but we need a good way to > >> measure performance. If we use this approach, we can omit it > >> for now and we can revisit your approach later without > >> breaking compatibility. How about using this approach if we > >> can't find a good way to measure performance? > > > > I think it would be better to hear more opinions about this idea and > > then make a decision, rather than basing our decision on whether or > > not we can measure its performance, so we can be more confident in the > > idea we have chosen. While this idea has the above downside, it could > > make sense because we always allocate the entire CopyFrom/ToStateData > > even today in spite of some fields being not used at all in binary > > format and it requires less implementation costs to hide the > > for-core-only fields. On the other hand, another possible idea is that > > we have three different structs for categories 1 (core-only), 2 (core > > and extensions), and 3 (extension-only), and expose only 2 that has a > > void pointer to 3. The core can allocate the memory for 1 that embeds > > 2 at the beginning of the fields. While this design looks cleaner and > > we can minimize overheads due to indirect references, it would require > > more implementation costs. Which method we choose, I think we need > > performance measurements in several scenarios to check if performance > > regressions don't happen unexpectedly. > > OK. So the next step is collecting more opinions, right? > > Could you add key people in this area to Cc to hear their > opinions? I'm not familiar with key people in the PostgreSQL > community... How about another idea like we move format-specific data to another struct that embeds CopyFrom/ToStateData at the first field and have CopyFrom/ToStart callback return memory with the size of that struct?It resolves the concerns about adding an extra indirection layer and extensions doesn't need to allocate memory for unnecessary fields (used only for built-in formats). While extensions can access the internal fields I think we can live with that given that there are some similar precedents such as table AM's scan descriptions. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-11 05:46 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-09-11 05:46 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBb3t7EcsjYT4w68p9OfMNwWTYsbSVaSRY6DRhi7sNRFg@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 10 Sep 2025 00:36:38 -0700, Masahiko Sawada <[email protected]> wrote: > How about another idea like we move format-specific data to another > struct that embeds CopyFrom/ToStateData at the first field and have > CopyFrom/ToStart callback return memory with the size of that > struct?It resolves the concerns about adding an extra indirection > layer and extensions doesn't need to allocate memory for unnecessary > fields (used only for built-in formats). While extensions can access > the internal fields I think we can live with that given that there are > some similar precedents such as table AM's scan descriptions. The another idea looks like the following, right? struct CopyToStateBuiltInData { struct CopyToStateData parent; /* Members for built-in formats */ ...; } typedef CopyToState *(*CopyToStart) (void); CopyToState BeginCopyTo(..., CopyToStart copy_to_start) { ...; /* Allocate workspace and zero all fields */ cstate = copy_to_start(); ...; } This idea will almost work. But we can't know which CopyToStart should be used before we parse "FORMAT" option of COPY. If we can iterate options twice in BeginCopy{To,From}(), we can know it. For example: BeginCopyTo(...) { ...; CopyToStart copy_to_start = NULL; foreach(option, options) { DefElem *defel = lfirst_node(DefElem, option); if (strcmp(defel->defname, "format") == 0) { char *fmt = defGetString(defel); if (strcmp(fmt, "text") == 0 || strcmp(fmt, "csv") == 0 || strcmp(fmt, "binary") == 0) { /* Use the builtin cstate */ } else { copy_to_start = /* Detect CopyToStart for custom format */; } } } if (copy_to_start) cstate = copy_to_start(); else cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateBuiltInData)); ...; } (It may be better that we add Copy{To,From}Routine::Copy{To,From}Allocate() instead of CopyToStart callback.) I think that this is acceptable because this must be a light process. This must not have negative performance impact. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-11 20:41 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-09-11 20:41 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Sep 10, 2025 at 10:46 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBb3t7EcsjYT4w68p9OfMNwWTYsbSVaSRY6DRhi7sNRFg@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 10 Sep 2025 00:36:38 -0700, > Masahiko Sawada <[email protected]> wrote: > > > How about another idea like we move format-specific data to another > > struct that embeds CopyFrom/ToStateData at the first field and have > > CopyFrom/ToStart callback return memory with the size of that > > struct?It resolves the concerns about adding an extra indirection > > layer and extensions doesn't need to allocate memory for unnecessary > > fields (used only for built-in formats). While extensions can access > > the internal fields I think we can live with that given that there are > > some similar precedents such as table AM's scan descriptions. > > The another idea looks like the following, right? > > struct CopyToStateBuiltInData > { > struct CopyToStateData parent; > > /* Members for built-in formats */ > ...; > } > > typedef CopyToState *(*CopyToStart) (void); > > CopyToState > BeginCopyTo(..., CopyToStart copy_to_start) > { > ...; > > /* Allocate workspace and zero all fields */ > cstate = copy_to_start(); > ...; > } Right. > This idea will almost work. But we can't know which > CopyToStart should be used before we parse "FORMAT" option > of COPY. > > If we can iterate options twice in BeginCopy{To,From}(), we > can know it. For example: > > BeginCopyTo(...) > { > ...; > > CopyToStart copy_to_start = NULL; > foreach(option, options) > { > DefElem *defel = lfirst_node(DefElem, option); > > if (strcmp(defel->defname, "format") == 0) > { > char *fmt = defGetString(defel); > if (strcmp(fmt, "text") == 0 || > strcmp(fmt, "csv") == 0 || > strcmp(fmt, "binary") == 0) { > /* Use the builtin cstate */ > } else { > copy_to_start = /* Detect CopyToStart for custom format */; > } > } > } > if (copy_to_start) > cstate = copy_to_start(); > else > cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateBuiltInData)); > ...; > } > > (It may be better that we add > Copy{To,From}Routine::Copy{To,From}Allocate() instead of > CopyToStart callback.) I think we can use a local variable of CopyFormatOptions and memcpy it to the opts of the returned cstate. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-12 00:07 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-09-12 00:07 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoCfqD=f2ELqPxg62+_QADhHi_kJXCDMhAerBtvxudd-xQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 11 Sep 2025 13:41:26 -0700, Masahiko Sawada <[email protected]> wrote: > I think we can use a local variable of CopyFormatOptions and memcpy it > to the opts of the returned cstate. It'll work too. Can we proceed this proposal with this approach? Should we collect more opinions before we proceed? If so, Could you add key people in this area to Cc to hear their opinions? Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-09-15 17:00 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-09-15 17:00 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Sep 11, 2025 at 5:07 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoCfqD=f2ELqPxg62+_QADhHi_kJXCDMhAerBtvxudd-xQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 11 Sep 2025 13:41:26 -0700, > Masahiko Sawada <[email protected]> wrote: > > > I think we can use a local variable of CopyFormatOptions and memcpy it > > to the opts of the returned cstate. > > It'll work too. Can we proceed this proposal with this > approach? Should we collect more opinions before we proceed? > If so, Could you add key people in this area to Cc to hear > their opinions? Since we don't have a single decision-maker, we should proceed through consensus-building and careful evaluation of each approach. I see that several senior hackers are already included in this thread, which is excellent. Since you and I, who have been involved in these discussions, agreed with this approach, I believe we can proceed with this direction. If anyone proposes alternative solutions that we find more compelling, we might have to change the approach. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-10-03 07:06 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-10-03 07:06 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoADXWgdizS0mV5w8wdfftDRsm8sUtNW=CzYYS1OhjFD2A@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Sep 2025 10:00:18 -0700, Masahiko Sawada <[email protected]> wrote: >> > I think we can use a local variable of CopyFormatOptions and memcpy it >> > to the opts of the returned cstate. >> >> It'll work too. Can we proceed this proposal with this >> approach? Should we collect more opinions before we proceed? >> If so, Could you add key people in this area to Cc to hear >> their opinions? > > Since we don't have a single decision-maker, we should proceed through > consensus-building and careful evaluation of each approach. I see that > several senior hackers are already included in this thread, which is > excellent. Since you and I, who have been involved in these > discussions, agreed with this approach, I believe we can proceed with > this direction. If anyone proposes alternative solutions that we find > more compelling, we might have to change the approach. OK. There is no objection for now. How about the attached patch? The patch uses the approach only for CopyToStateData. If this looks good, I can do it for CopyFromStateData too. This patch splits CopyToStateData to * CopyToStateData * CopyToStateInternalData * CopyToStateBuiltinData structs. This is based on the category described in https://www.postgresql.org/message-id/flat/CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC%2Bgf%3D3w7XiA4LQnvx0g%4... : > 1. fields used only the core (not by format callback) > 2. fields used by both the core and format callbacks > 3. built-in format specific fields (mostly for text and csv) CopyToStateInternalData is for 1. CopyToStateData is for 2. CopyToStateBuiltinData is for 3. This patch adds CopyToRoutine::CopyToGetStateSize() that returns size of state struct for the routine. For example, Built-in formats use sizeof(CopyToStateBuiltinData) for it. BeginCopyTo() allocates sizeof(CopyToStateInternalData) + CopyToGetStateSize() size continuous memory and uses the front part as CopyToStateInternalData and the back part as CopyToStateData/CopyToStateBuilinData. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-10-13 21:40 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-10-13 21:40 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; Andres Freund <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Oct 3, 2025 at 12:06 AM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoADXWgdizS0mV5w8wdfftDRsm8sUtNW=CzYYS1OhjFD2A@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Sep 2025 10:00:18 -0700, > Masahiko Sawada <[email protected]> wrote: > > >> > I think we can use a local variable of CopyFormatOptions and memcpy it > >> > to the opts of the returned cstate. > >> > >> It'll work too. Can we proceed this proposal with this > >> approach? Should we collect more opinions before we proceed? > >> If so, Could you add key people in this area to Cc to hear > >> their opinions? > > > > Since we don't have a single decision-maker, we should proceed through > > consensus-building and careful evaluation of each approach. I see that > > several senior hackers are already included in this thread, which is > > excellent. Since you and I, who have been involved in these > > discussions, agreed with this approach, I believe we can proceed with > > this direction. If anyone proposes alternative solutions that we find > > more compelling, we might have to change the approach. > > OK. There is no objection for now. > > How about the attached patch? The patch uses the approach > only for CopyToStateData. If this looks good, I can do it > for CopyFromStateData too. > > This patch splits CopyToStateData to > > * CopyToStateData > * CopyToStateInternalData > * CopyToStateBuiltinData > > structs. > > This is based on the category described in > https://www.postgresql.org/message-id/flat/CAD21AoBa0Wm3C2H12jaqkvLidP2zEhsC%2Bgf%3D3w7XiA4LQnvx0g%4... > : > > > 1. fields used only the core (not by format callback) > > 2. fields used by both the core and format callbacks > > 3. built-in format specific fields (mostly for text and csv) > > CopyToStateInternalData is for 1. > CopyToStateData is for 2. > CopyToStateBuiltinData is for 3. > > > This patch adds CopyToRoutine::CopyToGetStateSize() that > returns size of state struct for the routine. For example, > Built-in formats use sizeof(CopyToStateBuiltinData) for it. > > BeginCopyTo() allocates sizeof(CopyToStateInternalData) + > CopyToGetStateSize() size continuous memory and uses the > front part as CopyToStateInternalData and the back part as > CopyToStateData/CopyToStateBuilinData. Thank you for drafting the idea! The patch refactors the CopyToStateData so that we can both hide internal-use-only fields from extensions and extension can use its own state data, while not adding extra indirection layers. TBH I'm really not sure we must fully hide internal fields from extensions. Other extendable components seem not to strictly hide internal information from extensions. I'd suggest starting with only the latter point. That is, we merge fields in CopyToStateInternalData to CopyToStateData. What do you think? Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-10-14 02:15 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-10-14 02:15 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700, Masahiko Sawada <[email protected]> wrote: > The patch refactors the CopyToStateData so that we can both hide > internal-use-only fields from extensions and extension can use its own > state data, while not adding extra indirection layers. TBH I'm really > not sure we must fully hide internal fields from extensions. Other > extendable components seem not to strictly hide internal information > from extensions. I'd suggest starting with only the latter point. That > is, we merge fields in CopyToStateInternalData to CopyToStateData. > What do you think? OK. Let's follow the existing style. How about the attached patch? It merges CopyToStateInternalData to CopyToStateData. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-10-29 20:41 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-10-29 20:41 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Oct 13, 2025 at 7:15 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700, > Masahiko Sawada <[email protected]> wrote: > > > The patch refactors the CopyToStateData so that we can both hide > > internal-use-only fields from extensions and extension can use its own > > state data, while not adding extra indirection layers. TBH I'm really > > not sure we must fully hide internal fields from extensions. Other > > extendable components seem not to strictly hide internal information > > from extensions. I'd suggest starting with only the latter point. That > > is, we merge fields in CopyToStateInternalData to CopyToStateData. > > What do you think? > > OK. Let's follow the existing style. How about the attached > patch? It merges CopyToStateInternalData to CopyToStateData. > The basic idea of this patch makes sense to me. Andres, I believe that the current idea deals with your concerns about performance overheads. Particularly, we separate format-specific fields (c.f. CopyToStateBuiltinData struct in the patch) from the commonly-used fields (c.f., CopyToStateData struct), but the whole fields are stored in the contiguous memory. While the patch needs to be polished much, could you review if the basic idea of this patch addresses your concern? Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-11-14 20:19 Masahiko Sawada <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 2 replies; 127+ messages in thread From: Masahiko Sawada @ 2025-11-14 20:19 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Oct 29, 2025 at 1:41 PM Masahiko Sawada <[email protected]> wrote: > > On Mon, Oct 13, 2025 at 7:15 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAD21AoBkA=g=PN17r_iieru+vLyLtGZ8WvohgANa2vzsMfMogQ@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 13 Oct 2025 14:40:31 -0700, > > Masahiko Sawada <[email protected]> wrote: > > > > > The patch refactors the CopyToStateData so that we can both hide > > > internal-use-only fields from extensions and extension can use its own > > > state data, while not adding extra indirection layers. TBH I'm really > > > not sure we must fully hide internal fields from extensions. Other > > > extendable components seem not to strictly hide internal information > > > from extensions. I'd suggest starting with only the latter point. That > > > is, we merge fields in CopyToStateInternalData to CopyToStateData. > > > What do you think? > > > > OK. Let's follow the existing style. How about the attached > > patch? It merges CopyToStateInternalData to CopyToStateData. > > > > The basic idea of this patch makes sense to me. > This thread has involved extensive discussion, and the patch needs to be rebased. I'd like to summarize the current status of this patch and our discussions. I've attached updated patches that implement the whole ideas of this feature to help provide a clearer overall picture. In commits 2e4127b6d and 7717f6300, we refactored COPY TO/FROM code to use a set of callbacks for format-specific operations like data parsing. These callbacks are currently not exposed and are only used by built-in formats. The next step is to allow extensions to register their own format implementations, which these attached patches accomplish. The registration API can be called in _PG_init() as follows to enable users to specify a custom format name ('jsonlines' in this example) in the FORMAT option: RegisterCopyCustomFormat("jsonlines", &JsonLinesCopyFromRoutine, &JsonLinesCopyToRoutine); However, before introducing the registration API, we need to resolve an issue with how we currently use a monolithic struct (Copy{From,To}StateData) to store COPY TO/FROM state data. This struct currently contains both format-agnostic fields (e.g., target relation and source file) and format-specific fields (e.g., input buffers and EOL type). Patches 0001 and 0002 reorganize these fields to separate them. Specifically, format-specific fields are moved to a new struct while Copy{From,To}StateData retains format-agnostic fields, as shown here (e.g., COPY TO case): typedef struct CopyToStateData { /* format-specific routines */ const struct CopyToRoutine *routine; /* low-level state data */ CopyDest copy_dest; /* type of copy source/destination */ FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */ StringInfo fe_msgbuf; /* used for all dests during COPY TO */ (snip) /* * Working state */ MemoryContext copycontext; /* per-copy execution context */ FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ } CopyToStateData; typedef struct CopyToStateTextLike { CopyToStateData base; /* embedded */ int file_encoding; bool need_transcoding; bool encoding_embeds_ascii; } CopyToStateTextLike; Extensions must specify their required state struct size (like CopyToStateTextLike for built-in formats) using new callbacks Copy{From,To}EstimateStateSpace, allowing the core to allocate the appropriate amount of memory. This approach offers two advantages: - Format processing implementations only use the memory they need - No additional pointer traversal compared to using an opaque pointer for format-specific data Patches 0003 through 0006 implement the following: 0003: Introduces the RegisterCustomCopyFormat() API 0004 and 0005: Enable custom format implementations to register their own COPY command options 0006: Adds regression tests With these patches, here's what we can do using the 'jsonlines' format extension: -- COPY TO CREATE TABLE jl (id int, a text, b jsonb); INSERT INTO jl VALUES (1, 'hello', '{"test" : [1, true, {"num" : 42}]}'::jsonb), (2, 'hello world', 'true'), (999, null, '{"a" : 1}'); TABLE jl; id | a | b -----+-------------+---------------------------------- 1 | hello | {"test": [1, true, {"num": 42}]} 2 | hello world | true 999 | | {"a": 1} (3 rows) COPY jl TO '/tmp/test.jsonl' WITH (format 'jsonlines'); \! cat /tmp/test.jsonl {"id":1,"a":"hello","b":{"test": [1, true, {"num": 42}]}} {"id":2,"a":"hello world","b":true} {"id":999,"a":null,"b":{"a": 1}} -- COPY FROM CREATE TABLE jl_load (id int, a text, b jsonb); COPY jl_load FROM '/tmp/test.jsonl' WITH (format 'jsonlines'); -- COPY TO/FROM with custom options COPY jl TO '/tmp/jl.jsonl.gz' WITH (format 'jsonlines', compression 'gzip', compression_detail 'level=2'); COPY jl FROM '/tmp/jl.jsonl.gz' WITH (format 'jsonlines'); To demonstrate the functionality of both current and new APIs, Suto-san and I have created several experimental custom COPY format extensions: Apache Arrow (developed by Sutou-san): https://github.com/kou/pg-copy-arrow JSON Lines (developed by me): https://github.com/MasahikoSawada/pg_custom_copy_formats/blob/master/jsonlines.c These implementations serve as good examples of how extensions can use these APIs to define custom COPY formats. After offline discussions with Sutou-san, we believe the current APIs work well, particularly for text-based formats, though we still need to verify there are no performance regressions. One potential improvement would be adding support for random file access in COPY FROM operations. For example, with parquet files, it would be much more efficient to read the footer section first since it contains metadata, allowing selective reading of necessary file sections. The current sequential read API (CopyFromGetData()) requires reading all data to access the metadata. For future consideration, we could look into supporting file reading/writing from external sources like S3. While this is outside the scope of this patch, we discussed that allowing the core to delegate I/O operations to custom format implementations might be a good starting point. We can discuss this in a separate thread. I welcome your feedback on these proposed changes and APIs to help move this patch forward. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [application/octet-stream] 0003-Support-custom-COPY-format-for-COPY-FROM-and-COPY-TO.patch (9.6K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/2-0003-Support-custom-COPY-format-for-COPY-FROM-and-COPY-TO.patch) download | inline diff: From c1ab564adbe8db74c97c1d0e4ee6ec552457474c Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 10 Nov 2025 11:26:10 -0800 Subject: [PATCH 3/6] Support custom COPY format for COPY FROM and COPY TO. This commit enables extensions to register their own COPY format implementation for COPY TO and COPY FROM. If extensions register the custom format during _PG_init_ time, the format would be available for all backends whereas the backend can LOAD the custom format. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- src/backend/commands/Makefile | 1 + src/backend/commands/copy_custom_format.c | 142 ++++++++++++++++++++++ src/backend/commands/copyfrom.c | 7 ++ src/backend/commands/copyfromparse.c | 9 ++ src/backend/commands/copyto.c | 17 +++ src/backend/commands/meson.build | 1 + src/include/commands/copy.h | 6 + src/include/commands/copyapi.h | 9 ++ 8 files changed, 192 insertions(+) create mode 100644 src/backend/commands/copy_custom_format.c diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index f99acfd2b4b..74668d4ac9d 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -27,6 +27,7 @@ OBJS = \ copyfrom.o \ copyfromparse.o \ copyto.o \ + copy_custom_format.o \ createas.o \ dbcommands.o \ define.o \ diff --git a/src/backend/commands/copy_custom_format.c b/src/backend/commands/copy_custom_format.c new file mode 100644 index 00000000000..8bef6e779ac --- /dev/null +++ b/src/backend/commands/copy_custom_format.c @@ -0,0 +1,142 @@ +/*------------------------------------------------------------------------- + * + * copy_custom_format.c + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/copy_custom_format.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/copyapi.h" +#include "utils/memutils.h" + + +typedef struct CopyCustomFormat +{ + const char *fmt_name; + const CopyFromRoutine *from_routine; + const CopyToRoutine *to_routine; +} CopyCustomFormat; + +static CopyCustomFormat * CopyCustomFormatArray = NULL; +static int CopyCustomFormatAssigned = 0; +static int CopyCustomFormatAllocated = 0; + +static char *BuiltinFormatNames[] = { + "text", "csv", "binary" +}; + +void +RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine, + const CopyToRoutine *to_routine) +{ + CopyCustomFormat *entry; + + /* Check the duplication with built-in formats */ + for (int i = 0; i < lengthof(BuiltinFormatNames); i++) + { + if (strcmp(BuiltinFormatNames[i], fmt_name) == 0) + ereport(ERROR, + errmsg("failed to register custom COPY format \"%s\"", fmt_name), + errdetail("COPY format \"%s\" is a builtin format", + BuiltinFormatNames[i])); + + } + + /* Check the duplication with the registered custom formats */ + if (FindCustomCopyFormat(fmt_name)) + ereport(ERROR, + errmsg("failed to register custom COPY format \"%s\"", fmt_name), + errdetail("Custom COPY format \"%s\" already registered", + fmt_name)); + + /* If there is no array yet, create one */ + if (CopyCustomFormatArray == NULL) + { + CopyCustomFormatAllocated = 16; + CopyCustomFormatArray = (CopyCustomFormat *) + MemoryContextAlloc(TopMemoryContext, + CopyCustomFormatAllocated * sizeof(CopyCustomFormat)); + } + + /* If there's an array but it's current full, expand it */ + if (CopyCustomFormatAssigned >= CopyCustomFormatAllocated) + { + int i = pg_nextpower2_32(CopyCustomFormatAllocated + 1); + + CopyCustomFormatArray = (CopyCustomFormat *) + repalloc(CopyCustomFormatArray, i * sizeof(CopyCustomFormat)); + CopyCustomFormatAllocated = i; + } + + Assert(to_routine != NULL || from_routine != NULL); + Assert((to_routine == NULL) || + (to_routine->CopyToEstimateStateSpace != NULL && + to_routine->CopyToOutFunc != NULL && + to_routine->CopyToStart != NULL && + to_routine->CopyToOneRow != NULL && + to_routine->CopyToEnd != NULL)); + Assert((from_routine == NULL) || + (from_routine->CopyFromEstimateStateSpace != NULL && + from_routine->CopyFromInFunc != NULL && + from_routine->CopyFromStart != NULL && + from_routine->CopyFromOneRow != NULL && + from_routine->CopyFromEnd != NULL)); + + entry = &CopyCustomFormatArray[CopyCustomFormatAssigned++]; + entry->fmt_name = fmt_name; + entry->from_routine = from_routine; + entry->to_routine = to_routine; +} + +/* + * Returns true if the given custom format name is registered. + */ +bool +FindCustomCopyFormat(const char *fmt_name) +{ + for (int i = 0; i < CopyCustomFormatAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0) + return true; + } + + return false; +} + +bool +GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine) +{ + for (int i = 0; i < CopyCustomFormatAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0) + { + *to_routine = CopyCustomFormatArray[i].to_routine; + return true; + } + } + + return false; +} + +bool +GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine) +{ + for (int i = 0; i < CopyCustomFormatAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0) + { + *from_routine = CopyCustomFormatArray[i].from_routine; + return true; + } + } + + return false; +} diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 0c51e5ba5e1..592be4fcb5d 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1619,6 +1619,13 @@ create_copyfrom_state(ParseState *pstate, List *options) routine = &CopyFromRoutineCSV; else if (strcmp(fmt, "binary") == 0) routine = &CopyFromRoutineBinary; + else if (GetCustomCopyFromRoutine(fmt, &routine)) + { + if (routine == NULL) + ereport(ERROR, + errmsg("COPY format \"%s\" does not support COPY FROM", fmt), + parser_errposition(pstate, defel->location)); + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 669dbfb8459..30d8047f21a 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -228,6 +228,15 @@ ReceiveCopyBinaryHeader(CopyFromState ccstate) } } +/* + * Exporting CopyGetData() function for custom COPY FROM format implementations. + */ +int +CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread) +{ + return CopyGetData(cstate, databuf, minread, maxread); +} + /* * CopyGetData reads data from the source (file or frontend) * diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6a0a66507ba..b1b3ae141eb 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -435,6 +435,16 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } +/* + * Exporting CopySendEndOfRow() function for custom COPY TO format + * implementations. + */ +void +CopyToFlushData(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + static void CopySendEndOfRow(CopyToState cstate) { @@ -631,6 +641,13 @@ create_copyto_state(ParseState *pstate, List *options) routine = &CopyToRoutineCSV; else if (strcmp(fmt, "binary") == 0) routine = &CopyToRoutineBinary; + else if (GetCustomCopyToRoutine(fmt, &routine)) + { + if (routine == NULL) + ereport(ERROR, + errmsg("COPY format \"%s\" does not support COPY TO", fmt), + parser_errposition(pstate, defel->location)); + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build index 9f640ad4810..827f809eb53 100644 --- a/src/backend/commands/meson.build +++ b/src/backend/commands/meson.build @@ -15,6 +15,7 @@ backend_sources += files( 'copyfrom.c', 'copyfromparse.c', 'copyto.c', + 'copy_custom_format.c', 'createas.c', 'dbcommands.c', 'define.c', diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 30a1d2bff6e..82f07b05823 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -120,6 +120,12 @@ extern uint64 CopyFrom(CopyFromState cstate); extern DestReceiver *CreateCopyDestReceiver(void); +extern void ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out, + bool is_from, List **other_options, ParseState *pstate); + +/* defined in copy_custom_format.c */ +extern bool FindCustomCopyFormat(const char *fmt_name); + /* * internal prototypes */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c3d2199a0b6..2090a4e1c61 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -111,4 +111,13 @@ typedef struct CopyFromRoutine */ void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; + +extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread); +extern void CopyToFlushData(CopyToState cstate); + +extern void RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine, + const CopyToRoutine *to_routine); +extern bool GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine); +extern bool GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine); + #endif /* COPYAPI_H */ -- 2.47.3 [application/octet-stream] 0006-Add-regression-test-module-for-custom-COPY-format.patch (17.0K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/3-0006-Add-regression-test-module-for-custom-COPY-format.patch) download | inline diff: From 2718f015e44354dca5ba99927d6ef0aa2dbb3b1f Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 10 Nov 2025 22:18:28 -0800 Subject: [PATCH 6/6] Add regression test module for custom COPY format. --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + .../test_copy_custom_format/.gitignore | 3 + .../modules/test_copy_custom_format/Makefile | 21 ++ .../expected/test_copy_custom_format.out | 97 ++++++++ .../test_copy_custom_format/meson.build | 35 +++ .../sql/test_copy_custom_format.sql | 34 +++ .../test_copy_custom_format.c | 217 ++++++++++++++++++ .../test_copy_custom_format.conf | 1 + 9 files changed, 410 insertions(+) create mode 100644 src/test/modules/test_copy_custom_format/.gitignore create mode 100644 src/test/modules/test_copy_custom_format/Makefile create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out create mode 100644 src/test/modules/test_copy_custom_format/meson.build create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.conf diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 902a7954101..710ccb74990 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ test_bitmapset \ test_bloomfilter \ test_copy_callbacks \ + test_copy_custom_format \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 14fc761c4cf..5a2e4e03b80 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -18,6 +18,7 @@ subdir('test_binaryheap') subdir('test_bitmapset') subdir('test_bloomfilter') subdir('test_copy_callbacks') +subdir('test_copy_custom_format') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') diff --git a/src/test/modules/test_copy_custom_format/.gitignore b/src/test/modules/test_copy_custom_format/.gitignore new file mode 100644 index 00000000000..8d3af519eb1 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/.gitignore @@ -0,0 +1,3 @@ +/log/ +/results/ +/tmp_check/ \ No newline at end of file diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile new file mode 100644 index 00000000000..8ad1cebaba6 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/Makefile @@ -0,0 +1,21 @@ +# src/test/modules/test_copy_custom_format/Makefile + +MODULE_big = test_copy_custom_format +OBJS = \ + $(WIN32RES) \ + test_copy_custom_format.o +PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT" + +REGRESS = test_copy_custom_format +REGRESS_OPTS = --temp-config $(top_srcdir)/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_copy_custom_format +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out new file mode 100644 index 00000000000..a91ac293447 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out @@ -0,0 +1,97 @@ +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY copy_data FROM stdin WITH (FORMAT 'test_format'); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int 0 common_bool 0 from_str "(null)" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data TO stdout WITH (FORMAT 'test_format'); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes: 3 +NOTICE: common_int 0 common_bool 0 to_str "(null)" +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +-- Error +COPY copy_data FROM stdin WITH (FORMAT 'error'); +ERROR: COPY format "error" not recognized +LINE 1: COPY copy_data FROM stdin WITH (FORMAT 'error'); + ^ +COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error'); +ERROR: conflicting or redundant options +LINE 1: COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error... + ^ +-- Option handling, COPY FROM +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 10); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int 10 common_bool 0 from_str "(null)" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int -10); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int -10 common_bool 0 from_str "(null)" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 'a'); -- error +ERROR: common_int requires an integer value +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'true'); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int 0 common_bool 1 from_str "(null)" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'false'); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int 0 common_bool 0 from_str "(null)" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'hello'); -- error +ERROR: common_bool requires a Boolean value +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 100, common_bool false, from_str 'from option'); +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: common_int 100 common_bool 0 from_str "from option" +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'option'); -- error +ERROR: COPY format "invalid" not recognized +LINE 1: ... copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'o... + ^ +COPY copy_data FROM stdin WITH (FORMAT 'test_format', format 'text'); -- error +ERROR: conflicting or redundant options +LINE 1: ... copy_data FROM stdin WITH (FORMAT 'test_format', format 'te... + ^ +-- Option handling, COPY FROM +COPY copy_data TO stdout WITH (FORMAT 'test_format', common_int -10, common_bool false, to_str 'to option'); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes: 3 +NOTICE: common_int -10 common_bool 0 to_str "to option" +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (FORMAT 'test_format', from_str 'to option'); -- error +ERROR: COPY format "from_str" not recognized +LINE 1: ...Y copy_data TO stdout WITH (FORMAT 'test_format', from_str '... + ^ diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build new file mode 100644 index 00000000000..6cc6c8b60fd --- /dev/null +++ b/src/test/modules/test_copy_custom_format/meson.build @@ -0,0 +1,35 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +test_copy_custom_format_sources = files( +'test_copy_custom_format.c', +) + +if host_system == 'windows' + test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_copy_custom_format', + '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',]) +endif + +test_copy_custom_format = shared_module('test_copy_custom_format', + test_copy_custom_format_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_copy_custom_format + +tests += { + 'name': 'test_copy_custom_format', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_copy_custom_format', + ], + 'regress_args': [ + '--temp-config', files('test_copy_custom_format.conf'), + ], + # Disabled because these tests require + # "shared_preload_libraries=test_custom_copy_format", which typical + # runningcheck users do not have (e.g. buildfarm clients). + 'runningcheck': false, + }, +} diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql new file mode 100644 index 00000000000..b6564f0355b --- /dev/null +++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql @@ -0,0 +1,34 @@ +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); + +COPY copy_data FROM stdin WITH (FORMAT 'test_format'); +\. +COPY copy_data TO stdout WITH (FORMAT 'test_format'); + +-- Error +COPY copy_data FROM stdin WITH (FORMAT 'error'); +COPY copy_data FROM stdin WITH (FORMAT 'text', FORMAT 'error'); + + +-- Option handling, COPY FROM +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 10); +\. +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int -10); +\. +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 'a'); -- error + +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'true'); +\. +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'false'); +\. +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_bool 'hello'); -- error + +COPY copy_data FROM stdin WITH (FORMAT 'test_format', common_int 100, common_bool false, from_str 'from option'); +\. + +COPY copy_data FROM stdin WITH (FORMAT 'test_format', invalid 'option'); -- error +COPY copy_data FROM stdin WITH (FORMAT 'test_format', format 'text'); -- error + +-- Option handling, COPY FROM +COPY copy_data TO stdout WITH (FORMAT 'test_format', common_int -10, common_bool false, to_str 'to option'); +COPY copy_data TO stdout WITH (FORMAT 'test_format', from_str 'to option'); -- error diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c new file mode 100644 index 00000000000..a63390e875b --- /dev/null +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c @@ -0,0 +1,217 @@ +/*-------------------------------------------------------------------------- + * + * test_copy_custom_format.c + * Code for testing custom COPY format. + * + * Portions Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_copy_custom_format/test_copy_custom_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copystate.h" +#include "commands/copyapi.h" +#include "commands/defrem.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +typedef struct TestCopyCommonOption +{ + int common_int; + bool common_bool; +} TestCopyCommonOption; + +typedef struct TestCopyFromState +{ + CopyFromStateData base; + + TestCopyCommonOption common_opts; + char *from_str_option; +} TestCopyFromState; + +typedef struct TestCopyToState +{ + CopyToStateData base; + + TestCopyCommonOption common_opts; + char *to_str_option; +} TestCopyToState; + +static Size +TestCopyToEsimateSpace(void) +{ + return sizeof(TestCopyToState); +} + +static Size +TestCopyFromEsimateSpace(void) +{ + return sizeof(TestCopyFromState); +} + +static bool +TestCopyProcessCommonOption(TestCopyCommonOption * opt, DefElem *option) +{ + if (strcmp(option->defname, "common_int") == 0) + { + int val = defGetInt32(option); + + opt->common_int = val; + + return true; + } + else if (strcmp(option->defname, "common_bool") == 0) + { + bool val = defGetBoolean(option); + + opt->common_bool = val; + + return true; + } + + return false; +} + +static bool +TestCopyFromProcessOneOption(CopyFromState ccstate, DefElem *option) +{ + TestCopyFromState *cstate = (TestCopyFromState *) ccstate; + + if (TestCopyProcessCommonOption(&cstate->common_opts, option)) + { + return true; + } + else if (strcmp(option->defname, "from_str") == 0) + { + char *val = defGetString(option); + + cstate->from_str_option = val; + return true; + } + + return false; +} + +static bool +TestCopyToProcessOneOption(CopyToState ccstate, DefElem *option) +{ + TestCopyToState *cstate = (TestCopyToState *) ccstate; + + if (TestCopyProcessCommonOption(&cstate->common_opts, option)) + { + return true; + } + else if (strcmp(option->defname, "to_str") == 0) + { + char *val = defGetString(option); + + cstate->to_str_option = val; + return true; + } + + return false; +} + +static void +TestCopyFromInFunc(CopyFromState cstate, Oid atttypid, + FmgrInfo *finfo, Oid *typioparam) +{ + ereport(NOTICE, + errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +TestCopyFromStart(CopyFromState ccstate, TupleDesc tupDesc) +{ + TestCopyFromState *cstate = (TestCopyFromState *) ccstate; + + ereport(NOTICE, + errmsg("CopyFromStart: the number of attributes: %d", tupDesc->natts)); + ereport(NOTICE, + errmsg("common_int %d common_bool %d from_str \"%s\"", + cstate->common_opts.common_int, + cstate->common_opts.common_bool, + cstate->from_str_option)); +} + +static bool +TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls, + CopyFromRowInfo * rowinfo) +{ + ereport(NOTICE, + errmsg("CopyFromOneRow")); + + return false; +} + +static void +TestCopyFromEnd(CopyFromState cstate) +{ + ereport(NOTICE, + errmsg("CopyFromEnd")); +} + +static void +TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo) +{ + ereport(NOTICE, + errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +TestCopyToStart(CopyToState ccstate, TupleDesc tupDesc) +{ + TestCopyToState *cstate = (TestCopyToState *) ccstate; + + ereport(NOTICE, + errmsg("CopyToStart: the number of attributes: %d", tupDesc->natts)); + ereport(NOTICE, + errmsg("common_int %d common_bool %d to_str \"%s\"", + cstate->common_opts.common_int, + cstate->common_opts.common_bool, + cstate->to_str_option)); +} + +static void +TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid))); +} + +static void +TestCopyToEnd(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToEnd"))); +} + +static const CopyToRoutine CopyToRoutineTestCopyFormat = { + .CopyToEstimateStateSpace = TestCopyToEsimateSpace, + .CopyToProcessOneOption = TestCopyToProcessOneOption, + .CopyToOutFunc = TestCopyToOutFunc, + .CopyToStart = TestCopyToStart, + .CopyToOneRow = TestCopyToOneRow, + .CopyToEnd = TestCopyToEnd, +}; + + +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { + .CopyFromEstimateStateSpace = TestCopyFromEsimateSpace, + .CopyFromProcessOneOption = TestCopyFromProcessOneOption, + .CopyFromInFunc = TestCopyFromInFunc, + .CopyFromStart = TestCopyFromStart, + .CopyFromOneRow = TestCopyFromOneRow, + .CopyFromEnd = TestCopyFromEnd, +}; + +void +_PG_init(void) +{ + RegisterCopyCustomFormat("test_format", + &CopyFromRoutineTestCopyFormat, + &CopyToRoutineTestCopyFormat); +} diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf b/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf new file mode 100644 index 00000000000..fb7cf22a3b1 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.conf @@ -0,0 +1 @@ +shared_preload_libraries = 'test_copy_custom_format' -- 2.47.3 [application/octet-stream] 0004-Refactor-Copy-option-processing.patch (6.9K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/4-0004-Refactor-Copy-option-processing.patch) download | inline diff: From 362ceb242c7b026bcbb077c910b26512b08e1e9d Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 10 Nov 2025 14:53:51 -0800 Subject: [PATCH 4/6] Refactor Copy option processing. This commits refactors the COPY option handling to support custom COPY option callback. Since extensions can define custom COPY format with its own state data, it's necessary to pass the state data to the option process callback. This commits separates ProcessCopyOptions into ProcessCopyFromOptions and ProcessCopyToOptions and passes a pointer to Copy{From,To}State respectively. The subsequent patch introduces a new callback to allow extensions to define their own option processing callback. --- contrib/file_fdw/file_fdw.c | 2 +- src/backend/commands/copy.c | 18 ++++--------- src/backend/commands/copyfrom.c | 31 ++++++++++++++++++++++- src/backend/commands/copyto.c | 32 +++++++++++++++++++++++- src/include/commands/copyfrom_internal.h | 2 ++ 5 files changed, 69 insertions(+), 16 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index af2d86e5b5c..447fcb44241 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -338,7 +338,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Now apply the core COPY code's validation logic for more checks. */ - ProcessCopyOptions(NULL, NULL, true, other_options); + ProcessCopyFromOptions(NULL, other_options, NULL); /* * Either filename or program option is required for file_fdw foreign diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 28e878c3688..d4ee649c3c6 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -546,10 +546,8 @@ defGetCopyLogVerbosityChoice(DefElem *def, ParseState *pstate) * self-consistency of the options list. */ void -ProcessCopyOptions(ParseState *pstate, - CopyFormatOptions *opts_out, - bool is_from, - List *options) +ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out, + bool is_from, List **other_options, ParseState *pstate) { bool format_specified = false; bool freeze_specified = false; @@ -559,10 +557,6 @@ ProcessCopyOptions(ParseState *pstate, bool reject_limit_specified = false; ListCell *option; - /* Support external use for option sanity checking */ - if (opts_out == NULL) - opts_out = (CopyFormatOptions *) palloc0(sizeof(CopyFormatOptions)); - opts_out->file_encoding = -1; /* Extract options from the statement node tree */ @@ -583,6 +577,8 @@ ProcessCopyOptions(ParseState *pstate, opts_out->csv_mode = true; else if (strcmp(fmt, "binary") == 0) opts_out->binary = true; + else if (FindCustomCopyFormat(fmt)) + /* just validate option value */ ; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -731,11 +727,7 @@ ProcessCopyOptions(ParseState *pstate, opts_out->reject_limit = defGetCopyRejectLimitOption(defel); } else - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", - defel->defname), - parser_errposition(pstate, defel->location))); + *other_options = lappend(*other_options, defel); } /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 592be4fcb5d..6ca5466f244 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1705,7 +1705,7 @@ BeginCopyFrom(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options); + ProcessCopyFromOptions(cstate, options, pstate); /* Process the target relation */ cstate->rel = rel; @@ -2028,3 +2028,32 @@ ClosePipeFromProgram(CopyFromState cstate) errdetail_internal("%s", wait_result_to_str(pclose_rc)))); } } + +void +ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate) +{ + bool temp_state = false; + List *other_options = NIL; + CopyFormatOptions *opts; + + if (cstate == NULL) + { + cstate = create_copyfrom_state(pstate, options); + temp_state = true; + } + + opts = &cstate->opts; + + ProcessCopyBuiltinOptions(options, opts, true, &other_options, pstate); + + foreach_node(DefElem, option, other_options) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", option->defname), + parser_errposition(pstate, option->location))); + } + + if (temp_state) + pfree(cstate); +} diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b1b3ae141eb..ea31fa911f9 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -84,6 +84,7 @@ static void CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string bool use_quote); static void CopyRelationTo(CopyToState cstate, Relation rel, Relation root_rel, uint64 *processed); +static void ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate); /* built-in format-specific routines */ static Size CopyToEstimateStateTextLike(void); @@ -785,7 +786,7 @@ BeginCopyTo(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options); + ProcessCopyToOptions(cstate, options, pstate); /* Process the source/target relation or query */ if (rel) @@ -1570,3 +1571,32 @@ CreateCopyDestReceiver(void) return (DestReceiver *) self; } + +static void +ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate) +{ + bool temp_state = false; + List *other_options = NIL; + CopyFormatOptions *opts; + + if (cstate == NULL) + { + cstate = create_copyto_state(pstate, options); + temp_state = true; + } + + opts = &cstate->opts; + + ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate); + + foreach_node(DefElem, option, options) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", option->defname), + parser_errposition(pstate, option->location))); + } + + if (temp_state) + pfree(cstate); +} diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index d2d19536151..f457e2bee50 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -100,6 +100,8 @@ typedef struct CopyFromStateBuiltins extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); +extern void ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate); + /* One-row callbacks for built-in formats defined in copyfromparse.c */ extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); -- 2.47.3 [application/octet-stream] 0005-Add-Copy-To-From-ProcessOneOption-callback.patch (5.9K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/5-0005-Add-Copy-To-From-ProcessOneOption-callback.patch) download | inline diff: From 819a96d8cac5d41eae1dd2b32cdfd817199dd5ad Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 10 Nov 2025 15:05:05 -0800 Subject: [PATCH 5/6] Add Copy{To,From}ProcessOneOption callback. --- src/backend/commands/copyfrom.c | 15 +++++++++++---- src/backend/commands/copyto.c | 17 ++++++++++++----- src/include/commands/copyapi.h | 18 +++++++++++++++++- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 6ca5466f244..c23c4b1188a 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -132,6 +132,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate); /* text format */ static const CopyFromRoutine CopyFromRoutineText = { .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, + .CopyFromProcessOneOption = NULL, .CopyFromInFunc = CopyFromTextLikeInFunc, .CopyFromStart = CopyFromTextLikeStart, .CopyFromOneRow = CopyFromTextOneRow, @@ -141,6 +142,7 @@ static const CopyFromRoutine CopyFromRoutineText = { /* CSV format */ static const CopyFromRoutine CopyFromRoutineCSV = { .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, + .CopyFromProcessOneOption = NULL, .CopyFromInFunc = CopyFromTextLikeInFunc, .CopyFromStart = CopyFromTextLikeStart, .CopyFromOneRow = CopyFromCSVOneRow, @@ -150,6 +152,7 @@ static const CopyFromRoutine CopyFromRoutineCSV = { /* binary format */ static const CopyFromRoutine CopyFromRoutineBinary = { .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, + .CopyFromProcessOneOption = NULL, .CopyFromInFunc = CopyFromBinaryInFunc, .CopyFromStart = CopyFromBinaryStart, .CopyFromOneRow = CopyFromBinaryOneRow, @@ -2048,10 +2051,14 @@ ProcessCopyFromOptions(CopyFromState cstate, List *options, ParseState *pstate) foreach_node(DefElem, option, other_options) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", option->defname), - parser_errposition(pstate, option->location))); + if (cstate->routine->CopyFromProcessOneOption && + cstate->routine->CopyFromProcessOneOption(cstate, option)) + /* custom option is processed */ ; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", option->defname), + parser_errposition(pstate, option->location))); } if (temp_state) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index ea31fa911f9..9136098a0ed 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -122,6 +122,7 @@ static void CopySendInt16(CopyToState cstate, int16 val); /* text format */ static const CopyToRoutine CopyToRoutineText = { .CopyToEstimateStateSpace = CopyToEstimateStateTextLike, + .CopyToProcessOneOption = NULL, .CopyToStart = CopyToTextLikeStart, .CopyToOutFunc = CopyToTextLikeOutFunc, .CopyToOneRow = CopyToTextOneRow, @@ -131,6 +132,7 @@ static const CopyToRoutine CopyToRoutineText = { /* CSV format */ static const CopyToRoutine CopyToRoutineCSV = { .CopyToEstimateStateSpace = CopyToEstimateStateTextLike, + .CopyToProcessOneOption = NULL, .CopyToStart = CopyToTextLikeStart, .CopyToOutFunc = CopyToTextLikeOutFunc, .CopyToOneRow = CopyToCSVOneRow, @@ -140,6 +142,7 @@ static const CopyToRoutine CopyToRoutineCSV = { /* binary format */ static const CopyToRoutine CopyToRoutineBinary = { .CopyToEstimateStateSpace = CopyToEstimateStateBinary, + .CopyToProcessOneOption = NULL, .CopyToStart = CopyToBinaryStart, .CopyToOutFunc = CopyToBinaryOutFunc, .CopyToOneRow = CopyToBinaryOneRow, @@ -1589,12 +1592,16 @@ ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate) ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate); - foreach_node(DefElem, option, options) + foreach_node(DefElem, option, other_options) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", option->defname), - parser_errposition(pstate, option->location))); + if (cstate->routine->CopyToProcessOneOption && + cstate->routine->CopyToProcessOneOption(cstate, option)) + /* custom option is processed */ ; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", option->defname), + parser_errposition(pstate, option->location))); } if (temp_state) diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 2090a4e1c61..e19279d0c3d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -27,6 +27,14 @@ typedef struct CopyToRoutine */ Size (*CopyToEstimateStateSpace) (void); + /* + * Process one COPY TO option. Return true if the option is processed, + * otherwise return false. + * + * This is an optional callback. + */ + bool (*CopyToProcessOneOption) (CopyToState cstate, DefElem *option); + /* * Set output function information. This callback is called once at the * beginning of COPY TO. @@ -70,6 +78,14 @@ typedef struct CopyFromRoutine */ Size (*CopyFromEstimateStateSpace) (void); + /* + * Process one COPY FROM option. Return true if the option is processed, + * otherwise return false. + * + * This is an optional callback. + */ + bool (*CopyFromProcessOneOption) (CopyFromState cstate, DefElem *option); + /* * Set input function information. This callback is called once at the * beginning of COPY FROM. @@ -112,7 +128,7 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; -extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread); +extern int CopyFromGetData(CopyFromState cstate, void *databuf, int minread, int maxread); extern void CopyToFlushData(CopyToState cstate); extern void RegisterCopyCustomFormat(const char *fmt_name, const CopyFromRoutine *from_routine, -- 2.47.3 [application/octet-stream] 0002-Separate-format-specific-fields-from-CopyFromStateDa.patch (55.2K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/6-0002-Separate-format-specific-fields-from-CopyFromStateDa.patch) download | inline diff: From 54a307c3932d9fb080fdec3ce8b1068138f35f24 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Fri, 7 Nov 2025 15:33:32 -0800 Subject: [PATCH 2/6] Separate format-specific fields from CopyFromStateData struct. Previously, CopyFromStateData struct contained all state variables used throughout COPY FROM operations. To support the upcoming pluggable COPY format feature, this commit moves fields specific to built-in formats into their own dedicated struct. This commit also introduces a new CopyFromRoutine callback named CopyFromEstimateStateSpace, allowing format routines to estimate and return the memory size needed for their state data. The format's state data must embed CopyFromStateData as its first field, and the returned size must be equal to or larger than CopyFromStateData. While separate structs for core and format-specific usage might seem feasible, using contiguous memory avoids overhead from extra pointer traversal. This commit mixed many changes but the main goal is to move some fields that are used only for built-in formats from out of CopyFromStateData so that format implementation can use their own state data, similar to what we did for COPY TO. Here is the summary of changes: * Introduce a new callback CopyFromEstimateStateSpace. * Introduce a new struct CopyFromStateBuiltins for built-in format state data. * CopyFromStateData has fields used by the core. * The new struct embeds CopyFromStateData at the first field. * line_buf, raw_buf, and attribute_buf and their related fields are now format-specific fields. * Move some initialization steps (e.g., encoding convesion preparation) to format's CopyFromStart callback. Callback refactoring/handling. ============================== One concern is how to integrate the error callback with custom format. Currently, COPY FROM's error callback CopyFromErrorCallback() is set right before the main loop in CopyFrom() for COPY FROM command and unset after the loop, whereas COPY command API allows users to call just BeginCopyFrom(), NextCopyFrom(), and EndCopyFrom() (i.e., without CopyFrom()). I initially thought we can think that the error callback is format-specific function so we can set/unset the callback within the COPY format callbacks (e.g., setting the callback in CopyFromStart callback etc). However, I found out that it's not straightforward for example because CopyFromStart can be called multiple times by file_fdw.c. The current idea is that we allows format-implementations to update the core's cur_XXX fields so that the core's error callback can print the details. IOW, the error callback function is still a core function that allows format-implementation can access and update. While it looks working, one concern is priting the input row; now that the input buffer and line buffer are format-speicic fileds, there is no way for the core to access them to print the actual line being read. One trick I used is to have the format implementation set a pointer in the core's CopyFromStateData to the line_buf in format's CopyFromStateBuiltins. --- contrib/file_fdw/file_fdw.c | 5 +- src/backend/commands/copyfrom.c | 253 ++++++++++++-------- src/backend/commands/copyfromparse.c | 292 +++++++++++++---------- src/include/commands/copy.h | 12 +- src/include/commands/copyapi.h | 8 +- src/include/commands/copyfrom_internal.h | 104 +------- src/include/commands/copystate.h | 97 ++++++++ 7 files changed, 436 insertions(+), 335 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 70564a68b13..af2d86e5b5c 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -766,7 +766,8 @@ retry: */ ExecClearTuple(slot); - if (NextCopyFrom(cstate, econtext, slot->tts_values, slot->tts_isnull)) + if (NextCopyFrom(cstate, econtext, slot->tts_values, slot->tts_isnull, + NULL)) { if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE && cstate->escontext->error_occurred) @@ -1246,7 +1247,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, MemoryContextReset(tupcontext); MemoryContextSwitchTo(tupcontext); - found = NextCopyFrom(cstate, NULL, values, nulls); + found = NextCopyFrom(cstate, NULL, values, nulls, NULL); MemoryContextSwitchTo(oldcontext); diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 12781963b4f..0c51e5ba5e1 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -30,6 +30,8 @@ #include "catalog/namespace.h" #include "commands/copyapi.h" #include "commands/copyfrom_internal.h" +#include "commands/copystate.h" +#include "commands/defrem.h" #include "commands/progress.h" #include "commands/trigger.h" #include "executor/execPartition.h" @@ -129,6 +131,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate); /* text format */ static const CopyFromRoutine CopyFromRoutineText = { + .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, .CopyFromInFunc = CopyFromTextLikeInFunc, .CopyFromStart = CopyFromTextLikeStart, .CopyFromOneRow = CopyFromTextOneRow, @@ -137,6 +140,7 @@ static const CopyFromRoutine CopyFromRoutineText = { /* CSV format */ static const CopyFromRoutine CopyFromRoutineCSV = { + .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, .CopyFromInFunc = CopyFromTextLikeInFunc, .CopyFromStart = CopyFromTextLikeStart, .CopyFromOneRow = CopyFromCSVOneRow, @@ -145,54 +149,129 @@ static const CopyFromRoutine CopyFromRoutineCSV = { /* binary format */ static const CopyFromRoutine CopyFromRoutineBinary = { + .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, .CopyFromInFunc = CopyFromBinaryInFunc, .CopyFromStart = CopyFromBinaryStart, .CopyFromOneRow = CopyFromBinaryOneRow, .CopyFromEnd = CopyFromBinaryEnd, }; -/* Return a COPY FROM routine for the given options */ -static const CopyFromRoutine * -CopyFromGetRoutine(const CopyFormatOptions *opts) +/* + * Common routine to initialize CopyFromStateBuiltins data. + */ +static void +initialize_copyfrom_bultins_state(CopyFromStateBuiltins * state, TupleDesc tupDesc) { - if (opts->csv_mode) - return &CopyFromRoutineCSV; - else if (opts->binary) - return &CopyFromRoutineBinary; + /* Use client encoding when ENCODING option is not specified. */ + if (state->base.opts.file_encoding < 0) + state->file_encoding = pg_get_client_encoding(); + else + state->file_encoding = state->base.opts.file_encoding; + + /* + * Look up encoding conversion function. + */ + if (state->file_encoding == GetDatabaseEncoding() || + state->file_encoding == PG_SQL_ASCII || + GetDatabaseEncoding() == PG_SQL_ASCII) + { + state->need_transcoding = false; + } + else + { + state->need_transcoding = true; + state->conversion_proc = FindDefaultConversionProc(state->file_encoding, + GetDatabaseEncoding()); + if (!OidIsValid(state->conversion_proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(state->file_encoding), + pg_encoding_to_char(GetDatabaseEncoding())))); + } - /* default is text */ - return &CopyFromRoutineText; + state->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool)); + + /* initialize variables */ + state->eol_type = EOL_UNKNOWN; + + /* Convert convert_selectively name list to per-column flags */ + if (state->base.opts.convert_selectively) + { + List *attnums; + ListCell *cur; + int num_phys_attrs = RelationGetDescr(state->base.rel)->natts; + + state->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); + + attnums = CopyGetAttnums(tupDesc, state->base.rel, state->base.opts.convert_select); + + foreach(cur, attnums) + { + int attnum = lfirst_int(cur); + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + if (!list_member_int(state->base.attnumlist, attnum)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg_internal("selected column \"%s\" not referenced by COPY", + NameStr(attr->attname)))); + state->convert_select_flags[attnum - 1] = true; + } + } + + /* + * Allocate buffers for the input pipeline. + * + * attribute_buf and raw_buf are used in both text and binary modes, but + * input_buf and line_buf only in text mode. + */ + state->raw_buf = palloc(RAW_BUF_SIZE + 1); + state->raw_buf_index = state->raw_buf_len = 0; + + initStringInfo(&state->attribute_buf); + + /* Initialize state variables */ + state->base.cur_relname = RelationGetRelationName(state->base.rel); + state->base.cur_lineno = 0; + state->base.cur_attname = NULL; + state->base.cur_attval = NULL; + state->base.relname_only = false; } /* Implementation of the start callback for text and CSV formats */ static void CopyFromTextLikeStart(CopyFromState cstate, TupleDesc tupDesc) { + CopyFromStateBuiltins *state = (CopyFromStateBuiltins *) cstate; AttrNumber attr_count; + initialize_copyfrom_bultins_state(state, tupDesc); + /* * If encoding conversion is needed, we need another buffer to hold the * converted input data. Otherwise, we can just point input_buf to the * same buffer as raw_buf. */ - if (cstate->need_transcoding) + if (state->need_transcoding) { - cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); - cstate->input_buf_index = cstate->input_buf_len = 0; + state->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); + state->input_buf_index = state->input_buf_len = 0; } else - cstate->input_buf = cstate->raw_buf; - cstate->input_reached_eof = false; + state->input_buf = state->raw_buf; + state->input_reached_eof = false; - initStringInfo(&cstate->line_buf); + initStringInfo(&state->line_buf); + state->base.line_buf = &state->line_buf; /* * Create workspace for CopyReadAttributes results; used by CSV and text * format. */ - attr_count = list_length(cstate->attnumlist); - cstate->max_fields = attr_count; - cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); + attr_count = list_length(state->base.attnumlist); + state->max_fields = attr_count; + state->raw_fields = (char **) palloc(attr_count * sizeof(char *)); } /* @@ -220,6 +299,8 @@ CopyFromTextLikeEnd(CopyFromState cstate) static void CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc) { + initialize_copyfrom_bultins_state((CopyFromStateBuiltins *) cstate, tupDesc); + /* Read and verify binary header */ ReceiveCopyBinaryHeader(cstate); } @@ -308,7 +389,7 @@ CopyFromErrorCallback(void *arg) { char *lineval; - lineval = CopyLimitPrintoutLength(cstate->line_buf.data); + lineval = CopyLimitPrintoutLength(cstate->line_buf->data); errcontext("COPY %s, line %" PRIu64 ": \"%s\"", cstate->cur_relname, cstate->cur_lineno, lineval); @@ -1113,6 +1194,7 @@ CopyFrom(CopyFromState cstate) { TupleTableSlot *myslot; bool skip_tuple; + CopyFromRowInfo rowinfo = {0}; CHECK_FOR_INTERRUPTS(); @@ -1146,7 +1228,8 @@ CopyFrom(CopyFromState cstate) ExecClearTuple(myslot); /* Directly store the values/nulls array in the slot */ - if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull)) + if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull, + &rowinfo)) break; if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE && @@ -1379,8 +1462,8 @@ CopyFrom(CopyFromState cstate) /* Add this tuple to the tuple buffer */ CopyMultiInsertInfoStore(&multiInsertInfo, resultRelInfo, myslot, - cstate->line_buf.len, - cstate->cur_lineno); + rowinfo.tuplen, + rowinfo.lineno); /* * If enough inserts have queued up, then flush all @@ -1512,6 +1595,50 @@ CopyFrom(CopyFromState cstate) return processed; } +static CopyFromState +create_copyfrom_state(ParseState *pstate, List *options) +{ + const CopyFromRoutine *routine; + CopyFromState cstate; + Size req_size; + bool format_specified = false; + + routine = &CopyFromRoutineText; /* default */ + foreach_node(DefElem, defel, options) + { + if (strcmp(defel->defname, "format") == 0) + { + char *fmt = defGetString(defel); + + if (format_specified) + errorConflictingDefElem(defel, pstate); + format_specified = true; + if (strcmp(fmt, "text") == 0) + routine = &CopyFromRoutineText; + else if (strcmp(fmt, "csv") == 0) + routine = &CopyFromRoutineCSV; + else if (strcmp(fmt, "binary") == 0) + routine = &CopyFromRoutineBinary; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", fmt), + parser_errposition(pstate, defel->location))); + + break; + } + } + + req_size = routine->CopyFromEstimateStateSpace(); + Assert(req_size >= sizeof(CopyFromStateData)); + + /* Allocate workspace and zero all fields */ + cstate = (CopyFromState) palloc0(req_size); + cstate->routine = routine; + + return cstate; +} + /* * Setup to read tuples from a file for COPY FROM. * @@ -1558,7 +1685,7 @@ BeginCopyFrom(ParseState *pstate, }; /* Allocate workspace and zero all fields */ - cstate = (CopyFromStateData *) palloc0(sizeof(CopyFromStateData)); + cstate = create_copyfrom_state(pstate, options); /* * We allocate everything used by a cstate in a new memory context. This @@ -1573,9 +1700,6 @@ BeginCopyFrom(ParseState *pstate, /* Extract options from the statement node tree */ ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options); - /* Set the format routine */ - cstate->routine = CopyFromGetRoutine(&cstate->opts); - /* Process the target relation */ cstate->rel = rel; @@ -1657,82 +1781,10 @@ BeginCopyFrom(ParseState *pstate, } } - /* Convert convert_selectively name list to per-column flags */ - if (cstate->opts.convert_selectively) - { - List *attnums; - ListCell *cur; - - cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); - - attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.convert_select); - - foreach(cur, attnums) - { - int attnum = lfirst_int(cur); - Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - - if (!list_member_int(cstate->attnumlist, attnum)) - ereport(ERROR, - (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), - errmsg_internal("selected column \"%s\" not referenced by COPY", - NameStr(attr->attname)))); - cstate->convert_select_flags[attnum - 1] = true; - } - } - - /* Use client encoding when ENCODING option is not specified. */ - if (cstate->opts.file_encoding < 0) - cstate->file_encoding = pg_get_client_encoding(); - else - cstate->file_encoding = cstate->opts.file_encoding; - - /* - * Look up encoding conversion function. - */ - if (cstate->file_encoding == GetDatabaseEncoding() || - cstate->file_encoding == PG_SQL_ASCII || - GetDatabaseEncoding() == PG_SQL_ASCII) - { - cstate->need_transcoding = false; - } - else - { - cstate->need_transcoding = true; - cstate->conversion_proc = FindDefaultConversionProc(cstate->file_encoding, - GetDatabaseEncoding()); - if (!OidIsValid(cstate->conversion_proc)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_FUNCTION), - errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", - pg_encoding_to_char(cstate->file_encoding), - pg_encoding_to_char(GetDatabaseEncoding())))); - } - cstate->copy_src = COPY_FILE; /* default */ cstate->whereClause = whereClause; - /* Initialize state variables */ - cstate->eol_type = EOL_UNKNOWN; - cstate->cur_relname = RelationGetRelationName(cstate->rel); - cstate->cur_lineno = 0; - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; - cstate->relname_only = false; - - /* - * Allocate buffers for the input pipeline. - * - * attribute_buf and raw_buf are used in both text and binary modes, but - * input_buf and line_buf only in text mode. - */ - cstate->raw_buf = palloc(RAW_BUF_SIZE + 1); - cstate->raw_buf_index = cstate->raw_buf_len = 0; - cstate->raw_reached_eof = false; - - initStringInfo(&cstate->attribute_buf); - /* Assign range table and rteperminfos, we'll need them in CopyFrom. */ if (pstate) { @@ -1818,8 +1870,6 @@ BeginCopyFrom(ParseState *pstate, } } - cstate->defaults = (bool *) palloc0(tupDesc->natts * sizeof(bool)); - /* initialize progress */ pgstat_progress_start_command(PROGRESS_COMMAND_COPY, cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid); @@ -1833,6 +1883,7 @@ BeginCopyFrom(ParseState *pstate, cstate->volatile_defexprs = volatile_defexprs; cstate->num_defaults = num_defaults; cstate->is_program = is_program; + cstate->reached_eof = false; if (data_source_cb) { @@ -1959,7 +2010,7 @@ ClosePipeFromProgram(CopyFromState cstate) * should not report that as an error. Otherwise, SIGPIPE indicates a * problem. */ - if (!cstate->raw_reached_eof && + if (!cstate->reached_eof && wait_result_is_signal(pclose_rc, SIGPIPE)) return; diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index b1ae97b833d..669dbfb8459 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -138,21 +138,20 @@ if (1) \ /* NOTE: there's a copy of this in copyto.c */ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; - /* non-export function prototypes */ -static bool CopyReadLine(CopyFromState cstate, bool is_csv); -static bool CopyReadLineText(CopyFromState cstate, bool is_csv); -static int CopyReadAttributesText(CopyFromState cstate); -static int CopyReadAttributesCSV(CopyFromState cstate); -static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, +static bool CopyReadLine(CopyFromStateBuiltins * state, bool is_csv); +static bool CopyReadLineText(CopyFromStateBuiltins * state, bool is_csv); +static int CopyReadAttributesText(CopyFromStateBuiltins * state); +static int CopyReadAttributesCSV(CopyFromStateBuiltins * state); +static Datum CopyReadBinaryAttribute(CopyFromStateBuiltins * state, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); -static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate, +static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromStateBuiltins * state, ExprContext *econtext, - Datum *values, - bool *nulls, + Datum *values, bool *nulls, + CopyFromRowInfo * rowinfo, bool is_csv); -static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate, +static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromStateBuiltins * state, char ***fields, int *nfields, bool is_csv); @@ -161,10 +160,10 @@ static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromSta /* Low-level communications functions */ static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread); -static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); -static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); -static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); +static inline bool CopyGetInt32(CopyFromStateBuiltins * state, int32 *val); +static inline bool CopyGetInt16(CopyFromStateBuiltins * state, int16 *val); +static void CopyLoadInputBuf(CopyFromStateBuiltins * state); +static int CopyReadBinaryData(CopyFromStateBuiltins * state, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -187,8 +186,9 @@ ReceiveCopyBegin(CopyFromState cstate) } void -ReceiveCopyBinaryHeader(CopyFromState cstate) +ReceiveCopyBinaryHeader(CopyFromState ccstate) { + CopyFromStateBuiltins *cstate = (CopyFromStateBuiltins *) ccstate; char readSig[11]; int32 tmp; @@ -255,10 +255,10 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) (errcode_for_file_access(), errmsg("could not read from COPY file: %m"))); if (bytesread == 0) - cstate->raw_reached_eof = true; + cstate->reached_eof = true; break; case COPY_FRONTEND: - while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) + while (maxread > 0 && bytesread < minread && !cstate->reached_eof) { int avail; @@ -309,7 +309,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) break; case PqMsg_CopyDone: /* COPY IN correctly terminated by frontend */ - cstate->raw_reached_eof = true; + cstate->reached_eof = true; return bytesread; case PqMsg_CopyFail: ereport(ERROR, @@ -359,7 +359,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) * Returns true if OK, false if EOF */ static inline bool -CopyGetInt32(CopyFromState cstate, int32 *val) +CopyGetInt32(CopyFromStateBuiltins * cstate, int32 *val) { uint32 buf; @@ -376,7 +376,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) * CopyGetInt16 reads an int16 that appears in network byte order */ static inline bool -CopyGetInt16(CopyFromState cstate, int16 *val) +CopyGetInt16(CopyFromStateBuiltins * cstate, int16 *val) { uint16 buf; @@ -397,7 +397,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) * On entry, there must be some data to convert in 'raw_buf'. */ static void -CopyConvertBuf(CopyFromState cstate) +CopyConvertBuf(CopyFromStateBuiltins * cstate) { /* * If the file and server encoding are the same, no encoding conversion is @@ -421,7 +421,7 @@ CopyConvertBuf(CopyFromState cstate) /* * If no more raw data is coming, report the EOF to the caller. */ - if (cstate->raw_reached_eof) + if (cstate->base.reached_eof) cstate->input_reached_eof = true; return; } @@ -444,7 +444,7 @@ CopyConvertBuf(CopyFromState cstate) * least one character, and a failure to do so means that we've * hit an invalid byte sequence. */ - if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding)) + if (cstate->base.reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding)) cstate->input_reached_error = true; return; } @@ -467,7 +467,7 @@ CopyConvertBuf(CopyFromState cstate) /* * If no more raw data is coming, report the EOF to the caller. */ - if (cstate->raw_reached_eof) + if (cstate->base.reached_eof) cstate->input_reached_eof = true; return; } @@ -517,7 +517,7 @@ CopyConvertBuf(CopyFromState cstate) * failure to do so must mean that we've hit a byte sequence * that's invalid. */ - if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH) + if (cstate->base.reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH) cstate->input_reached_error = true; return; } @@ -530,7 +530,7 @@ CopyConvertBuf(CopyFromState cstate) * Report an encoding or conversion error. */ static void -CopyConversionError(CopyFromState cstate) +CopyConversionError(CopyFromStateBuiltins * cstate) { Assert(cstate->raw_buf_len > 0); Assert(cstate->input_reached_error); @@ -587,7 +587,7 @@ CopyConversionError(CopyFromState cstate) * beginning of the buffer, and we load new data after that. */ static void -CopyLoadRawBuf(CopyFromState cstate) +CopyLoadRawBuf(CopyFromStateBuiltins * cstate) { int nbytes; int inbytes; @@ -624,17 +624,17 @@ CopyLoadRawBuf(CopyFromState cstate) } /* Load more data */ - inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len, + inbytes = CopyGetData((CopyFromState) cstate, cstate->raw_buf + cstate->raw_buf_len, 1, RAW_BUF_SIZE - cstate->raw_buf_len); nbytes += inbytes; cstate->raw_buf[nbytes] = '\0'; cstate->raw_buf_len = nbytes; - cstate->bytes_processed += inbytes; - pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed); + cstate->base.bytes_processed += inbytes; + pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->base.bytes_processed); if (inbytes == 0) - cstate->raw_reached_eof = true; + cstate->base.reached_eof = true; } /* @@ -647,7 +647,7 @@ CopyLoadRawBuf(CopyFromState cstate) * of the buffer and then we load more data after that. */ static void -CopyLoadInputBuf(CopyFromState cstate) +CopyLoadInputBuf(CopyFromStateBuiltins * cstate) { int nbytes = INPUT_BUF_BYTES(cstate); @@ -685,7 +685,7 @@ CopyLoadInputBuf(CopyFromState cstate) break; /* Try to load more raw data */ - Assert(!cstate->raw_reached_eof); + Assert(!cstate->base.reached_eof); CopyLoadRawBuf(cstate); } } @@ -698,7 +698,7 @@ CopyLoadInputBuf(CopyFromState cstate) * would be less than 'nbytes' only if we reach EOF). */ static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +CopyReadBinaryData(CopyFromStateBuiltins * cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -723,7 +723,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) if (RAW_BUF_BYTES(cstate) == 0) { CopyLoadRawBuf(cstate); - if (cstate->raw_reached_eof) + if (cstate->base.reached_eof) break; /* EOF */ } @@ -739,15 +739,21 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +Size +CopyFromBuiltinsEstimateSpace(void) +{ + return sizeof(CopyFromStateBuiltins); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. */ bool -NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) +NextCopyFromRawFields(CopyFromState ccstate, char ***fields, int *nfields) { - return NextCopyFromRawFieldsInternal(cstate, fields, nfields, - cstate->opts.csv_mode); + return NextCopyFromRawFieldsInternal((CopyFromStateBuiltins *) ccstate, + fields, nfields, ccstate->opts.csv_mode); } /* @@ -768,35 +774,35 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) * by internal functions such as CopyFromTextLikeOneRow(). */ static pg_attribute_always_inline bool -NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv) +NextCopyFromRawFieldsInternal(CopyFromStateBuiltins * cstate, char ***fields, int *nfields, bool is_csv) { int fldct; bool done = false; /* only available for text or csv input */ - Assert(!cstate->opts.binary); + Assert(!cstate->base.opts.binary); /* on input check that the header line is correct if needed */ - if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE) + if (cstate->base.cur_lineno == 0 && cstate->base.opts.header_line != COPY_HEADER_FALSE) { ListCell *cur; TupleDesc tupDesc; - int lines_to_skip = cstate->opts.header_line; + int lines_to_skip = cstate->base.opts.header_line; /* If set to "match", one header line is skipped */ - if (cstate->opts.header_line == COPY_HEADER_MATCH) + if (cstate->base.opts.header_line == COPY_HEADER_MATCH) lines_to_skip = 1; - tupDesc = RelationGetDescr(cstate->rel); + tupDesc = RelationGetDescr(cstate->base.rel); for (int i = 0; i < lines_to_skip; i++) { - cstate->cur_lineno++; + cstate->base.cur_lineno++; if ((done = CopyReadLine(cstate, is_csv))) break; } - if (cstate->opts.header_line == COPY_HEADER_MATCH) + if (cstate->base.opts.header_line == COPY_HEADER_MATCH) { int fldnum; @@ -805,14 +811,14 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields else fldct = CopyReadAttributesText(cstate); - if (fldct != list_length(cstate->attnumlist)) + if (fldct != list_length(cstate->base.attnumlist)) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("wrong number of fields in header line: got %d, expected %d", - fldct, list_length(cstate->attnumlist)))); + fldct, list_length(cstate->base.attnumlist)))); fldnum = 0; - foreach(cur, cstate->attnumlist) + foreach(cur, cstate->base.attnumlist) { int attnum = lfirst_int(cur); char *colName; @@ -825,7 +831,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"", - fldnum, cstate->opts.null_print, NameStr(attr->attname)))); + fldnum, cstate->base.opts.null_print, NameStr(attr->attname)))); if (namestrcmp(&attr->attname, colName) != 0) { @@ -841,7 +847,7 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields return false; } - cstate->cur_lineno++; + cstate->base.cur_lineno++; /* Actually read the line into memory here */ done = CopyReadLine(cstate, is_csv); @@ -877,8 +883,8 @@ NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields * relation passed to BeginCopyFrom. This function fills the arrays. */ bool -NextCopyFrom(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls) +NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Datum *values, + bool *nulls, CopyFromRowInfo * rowinfo) { TupleDesc tupDesc; AttrNumber num_phys_attrs, @@ -893,10 +899,9 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, /* Initialize all values for row to NULL */ MemSet(values, 0, num_phys_attrs * sizeof(Datum)); MemSet(nulls, true, num_phys_attrs * sizeof(bool)); - MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); /* Get one row from source */ - if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls)) + if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls, rowinfo)) return false; /* @@ -923,17 +928,19 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, /* Implementation of the per-row callback for text format */ bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, - bool *nulls) + bool *nulls, CopyFromRowInfo * rowinfo) { - return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false); + return CopyFromTextLikeOneRow((CopyFromStateBuiltins *) cstate, econtext, + values, nulls, rowinfo, false); } /* Implementation of the per-row callback for CSV format */ bool CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, - bool *nulls) + bool *nulls, CopyFromRowInfo * rowinfo) { - return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true); + return CopyFromTextLikeOneRow((CopyFromStateBuiltins *) cstate, econtext, + values, nulls, rowinfo, true); } /* @@ -943,22 +950,25 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, * and to help compilers to optimize away the 'is_csv' condition. */ static pg_attribute_always_inline bool -CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls, bool is_csv) +CopyFromTextLikeOneRow(CopyFromStateBuiltins * cstate, ExprContext *econtext, + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo, + bool is_csv) { TupleDesc tupDesc; AttrNumber attr_count; - FmgrInfo *in_functions = cstate->in_functions; - Oid *typioparams = cstate->typioparams; - ExprState **defexprs = cstate->defexprs; + FmgrInfo *in_functions = cstate->base.in_functions; + Oid *typioparams = cstate->base.typioparams; + ExprState **defexprs = cstate->base.defexprs; char **field_strings; ListCell *cur; int fldct; int fieldno; char *string; - tupDesc = RelationGetDescr(cstate->rel); - attr_count = list_length(cstate->attnumlist); + tupDesc = RelationGetDescr(cstate->base.rel); + attr_count = list_length(cstate->base.attnumlist); + + MemSet(cstate->defaults, false, tupDesc->natts * sizeof(bool)); /* read raw fields in the next line */ if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv)) @@ -973,7 +983,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, fieldno = 0; /* Loop to read the user attributes on the line. */ - foreach(cur, cstate->attnumlist) + foreach(cur, cstate->base.attnumlist) { int attnum = lfirst_int(cur); int m = attnum - 1; @@ -996,16 +1006,16 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, if (is_csv) { if (string == NULL && - cstate->opts.force_notnull_flags[m]) + cstate->base.opts.force_notnull_flags[m]) { /* * FORCE_NOT_NULL option is set and column is NULL - convert * it to the NULL string. */ - string = cstate->opts.null_print; + string = cstate->base.opts.null_print; } - else if (string != NULL && cstate->opts.force_null_flags[m] - && strcmp(string, cstate->opts.null_print) == 0) + else if (string != NULL && cstate->base.opts.force_null_flags[m] + && strcmp(string, cstate->base.opts.null_print) == 0) { /* * FORCE_NULL option is set and column matches the NULL @@ -1017,8 +1027,8 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, } } - cstate->cur_attname = NameStr(att->attname); - cstate->cur_attval = string; + cstate->base.cur_attname = NameStr(att->attname); + cstate->base.cur_attval = string; if (string != NULL) nulls[m] = false; @@ -1039,73 +1049,81 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, string, typioparams[m], att->atttypmod, - (Node *) cstate->escontext, + (Node *) cstate->base.escontext, &values[m])) { - Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + Assert(cstate->base.opts.on_error != COPY_ON_ERROR_STOP); - cstate->num_errors++; + cstate->base.num_errors++; - if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) + if (cstate->base.opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) { /* * Since we emit line number and column info in the below * notice message, we suppress error context information other * than the relation name. */ - Assert(!cstate->relname_only); - cstate->relname_only = true; + Assert(!cstate->base.relname_only); + cstate->base.relname_only = true; - if (cstate->cur_attval) + if (cstate->base.cur_attval) { char *attval; - attval = CopyLimitPrintoutLength(cstate->cur_attval); + attval = CopyLimitPrintoutLength(cstate->base.cur_attval); ereport(NOTICE, errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", - cstate->cur_lineno, - cstate->cur_attname, + cstate->base.cur_lineno, + cstate->base.cur_attname, attval)); pfree(attval); } else ereport(NOTICE, errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", - cstate->cur_lineno, - cstate->cur_attname)); + cstate->base.cur_lineno, + cstate->base.cur_attname)); /* reset relname_only */ - cstate->relname_only = false; + cstate->base.relname_only = false; } return true; } - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; + cstate->base.cur_attname = NULL; + cstate->base.cur_attval = NULL; } Assert(fieldno == attr_count); + /* Set output parameters */ + if (rowinfo) + { + rowinfo->lineno = cstate->base.cur_lineno; + rowinfo->tuplen = cstate->line_buf.len; + } + return true; } /* Implementation of the per-row callback for binary format */ bool -CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, - bool *nulls) +CopyFromBinaryOneRow(CopyFromState ccstate, ExprContext *econtext, Datum *values, + bool *nulls, CopyFromRowInfo * rowinfo) { + CopyFromStateBuiltins *cstate = (CopyFromStateBuiltins *) ccstate; TupleDesc tupDesc; AttrNumber attr_count; - FmgrInfo *in_functions = cstate->in_functions; - Oid *typioparams = cstate->typioparams; + FmgrInfo *in_functions = cstate->base.in_functions; + Oid *typioparams = cstate->base.typioparams; int16 fld_count; ListCell *cur; - tupDesc = RelationGetDescr(cstate->rel); - attr_count = list_length(cstate->attnumlist); + tupDesc = RelationGetDescr(cstate->base.rel); + attr_count = list_length(cstate->base.attnumlist); - cstate->cur_lineno++; + cstate->base.cur_lineno++; if (!CopyGetInt16(cstate, &fld_count)) { @@ -1138,19 +1156,29 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, errmsg("row field count is %d, expected %d", (int) fld_count, attr_count))); - foreach(cur, cstate->attnumlist) + foreach(cur, cstate->base.attnumlist) { int attnum = lfirst_int(cur); int m = attnum - 1; Form_pg_attribute att = TupleDescAttr(tupDesc, m); - cstate->cur_attname = NameStr(att->attname); + cstate->base.cur_attname = NameStr(att->attname); values[m] = CopyReadBinaryAttribute(cstate, &in_functions[m], typioparams[m], att->atttypmod, &nulls[m]); - cstate->cur_attname = NULL; + cstate->base.cur_attname = NULL; + } + + if (rowinfo) + { + /* + * XXX: We used to use line_buf.len but we don't actually use line_buf + * in binary format. + */ + rowinfo->lineno = cstate->base.cur_lineno; + rowinfo->tuplen = cstate->line_buf.len; } return true; @@ -1164,12 +1192,12 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, * in the final value of line_buf. */ static bool -CopyReadLine(CopyFromState cstate, bool is_csv) +CopyReadLine(CopyFromStateBuiltins * cstate, bool is_csv) { bool result; resetStringInfo(&cstate->line_buf); - cstate->line_buf_valid = false; + cstate->base.line_buf_valid = false; /* Parse data and transfer into line_buf */ result = CopyReadLineText(cstate, is_csv); @@ -1181,13 +1209,13 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (cstate->base.copy_src == COPY_FRONTEND) { int inbytes; do { - inbytes = CopyGetData(cstate, cstate->input_buf, + inbytes = CopyGetData((CopyFromState) cstate, cstate->input_buf, 1, INPUT_BUF_SIZE); } while (inbytes > 0); cstate->input_buf_index = 0; @@ -1231,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) } /* Now it's safe to use the buffer in error messages */ - cstate->line_buf_valid = true; + cstate->base.line_buf_valid = true; return result; } @@ -1240,7 +1268,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * CopyReadLineText - inner loop of CopyReadLine for text mode */ static bool -CopyReadLineText(CopyFromState cstate, bool is_csv) +CopyReadLineText(CopyFromStateBuiltins * cstate, bool is_csv) { char *copy_input_buf; int input_buf_ptr; @@ -1257,8 +1285,8 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) if (is_csv) { - quotec = cstate->opts.quote[0]; - escapec = cstate->opts.escape[0]; + quotec = cstate->base.opts.quote[0]; + escapec = cstate->base.opts.escape[0]; /* ignore special escape processing if it's the same as quotec */ if (quotec == escapec) escapec = '\0'; @@ -1367,7 +1395,7 @@ CopyReadLineText(CopyFromState cstate, bool is_csv) * at all --- is cur_lineno a physical or logical count?) */ if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r')) - cstate->cur_lineno++; + cstate->base.cur_lineno++; } /* Process \r */ @@ -1570,9 +1598,9 @@ GetDecimalFromHex(char hex) * The return value is the number of fields actually read. */ static int -CopyReadAttributesText(CopyFromState cstate) +CopyReadAttributesText(CopyFromStateBuiltins * cstate) { - char delimc = cstate->opts.delim[0]; + char delimc = cstate->base.opts.delim[0]; int fieldno; char *output_ptr; char *cur_ptr; @@ -1755,26 +1783,26 @@ CopyReadAttributesText(CopyFromState cstate) /* Check whether raw input matched null marker */ input_len = end_ptr - start_ptr; - if (input_len == cstate->opts.null_print_len && - strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) + if (input_len == cstate->base.opts.null_print_len && + strncmp(start_ptr, cstate->base.opts.null_print, input_len) == 0) cstate->raw_fields[fieldno] = NULL; /* Check whether raw input matched default marker */ - else if (fieldno < list_length(cstate->attnumlist) && - cstate->opts.default_print && - input_len == cstate->opts.default_print_len && - strncmp(start_ptr, cstate->opts.default_print, input_len) == 0) + else if (fieldno < list_length(cstate->base.attnumlist) && + cstate->base.opts.default_print && + input_len == cstate->base.opts.default_print_len && + strncmp(start_ptr, cstate->base.opts.default_print, input_len) == 0) { /* fieldno is 0-indexed and attnum is 1-indexed */ - int m = list_nth_int(cstate->attnumlist, fieldno) - 1; + int m = list_nth_int(cstate->base.attnumlist, fieldno) - 1; - if (cstate->defexprs[m] != NULL) + if (cstate->base.defexprs[m] != NULL) { /* defaults contain entries for all physical attributes */ cstate->defaults[m] = true; } else { - TupleDesc tupDesc = RelationGetDescr(cstate->rel); + TupleDesc tupDesc = RelationGetDescr(cstate->base.rel); Form_pg_attribute att = TupleDescAttr(tupDesc, m); ereport(ERROR, @@ -1824,11 +1852,11 @@ CopyReadAttributesText(CopyFromState cstate) * "standard" (i.e. common) CSV usage. */ static int -CopyReadAttributesCSV(CopyFromState cstate) +CopyReadAttributesCSV(CopyFromStateBuiltins * cstate) { - char delimc = cstate->opts.delim[0]; - char quotec = cstate->opts.quote[0]; - char escapec = cstate->opts.escape[0]; + char delimc = cstate->base.opts.delim[0]; + char quotec = cstate->base.opts.quote[0]; + char escapec = cstate->base.opts.escape[0]; int fieldno; char *output_ptr; char *cur_ptr; @@ -1970,26 +1998,26 @@ endfield: /* Check whether raw input matched null marker */ input_len = end_ptr - start_ptr; - if (!saw_quote && input_len == cstate->opts.null_print_len && - strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) + if (!saw_quote && input_len == cstate->base.opts.null_print_len && + strncmp(start_ptr, cstate->base.opts.null_print, input_len) == 0) cstate->raw_fields[fieldno] = NULL; /* Check whether raw input matched default marker */ - else if (fieldno < list_length(cstate->attnumlist) && - cstate->opts.default_print && - input_len == cstate->opts.default_print_len && - strncmp(start_ptr, cstate->opts.default_print, input_len) == 0) + else if (fieldno < list_length(cstate->base.attnumlist) && + cstate->base.opts.default_print && + input_len == cstate->base.opts.default_print_len && + strncmp(start_ptr, cstate->base.opts.default_print, input_len) == 0) { /* fieldno is 0-index and attnum is 1-index */ - int m = list_nth_int(cstate->attnumlist, fieldno) - 1; + int m = list_nth_int(cstate->base.attnumlist, fieldno) - 1; - if (cstate->defexprs[m] != NULL) + if (cstate->base.defexprs[m] != NULL) { /* defaults contain entries for all physical attributes */ cstate->defaults[m] = true; } else { - TupleDesc tupDesc = RelationGetDescr(cstate->rel); + TupleDesc tupDesc = RelationGetDescr(cstate->base.rel); Form_pg_attribute att = TupleDescAttr(tupDesc, m); ereport(ERROR, @@ -2019,7 +2047,7 @@ endfield: * Read a binary attribute */ static Datum -CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, +CopyReadBinaryAttribute(CopyFromStateBuiltins * cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull) { diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index d75a70715a4..30a1d2bff6e 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -48,6 +48,12 @@ typedef enum CopyLogVerbosityChoice COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */ } CopyLogVerbosityChoice; +typedef struct CopyFromRowInfo +{ + uint64 lineno; + int tuplen; +} CopyFromRowInfo; + /* * A struct to hold COPY options, in a parsed form. All of these are related * to formatting, except for 'freeze', which doesn't really belong here, but @@ -89,8 +95,6 @@ typedef struct CopyFormatOptions /* defined in copystate.h */ typedef struct CopyToStateData *CopyToState; - -/* Private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); @@ -105,8 +109,8 @@ extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *where const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options); extern void EndCopyFrom(CopyFromState cstate); -extern bool NextCopyFrom(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); +extern bool NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Datum *values, + bool *nulls, CopyFromRowInfo * rowinfo); extern bool NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields); extern void CopyFromErrorCallback(void *arg); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 3b9982d54b8..c3d2199a0b6 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -65,6 +65,11 @@ typedef struct CopyToRoutine */ typedef struct CopyFromRoutine { + /* + * Estimate and return the memory size required to store the state data. + */ + Size (*CopyFromEstimateStateSpace) (void); + /* * Set input function information. This callback is called once at the * beginning of COPY FROM. @@ -99,12 +104,11 @@ typedef struct CopyFromRoutine * Returns false if there are no more tuples to read. */ bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); /* * End a COPY FROM. This callback is called once at the end of COPY FROM. */ void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; - #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index c8b22af22d8..d2d19536151 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -15,105 +15,23 @@ #define COPYFROM_INTERNAL_H #include "commands/copy.h" -#include "commands/trigger.h" -#include "nodes/miscnodes.h" +#include "commands/copystate.h" /* - * Represents the different source cases we need to worry about at - * the bottom level + * COPY FROM state data used for builtin formats. */ -typedef enum CopySource +typedef struct CopyFromStateBuiltins { - COPY_FILE, /* from file (or a piped program) */ - COPY_FRONTEND, /* from frontend */ - COPY_CALLBACK, /* from callback function */ -} CopySource; - -/* - * Represents the end-of-line terminator type of the input - */ -typedef enum EolType -{ - EOL_UNKNOWN, - EOL_NL, - EOL_CR, - EOL_CRNL, -} EolType; - -/* - * Represents the insert method to be used during COPY FROM. - */ -typedef enum CopyInsertMethod -{ - CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */ - CIM_MULTI, /* always use table_multi_insert or - * ExecForeignBatchInsert */ - CIM_MULTI_CONDITIONAL, /* use table_multi_insert or - * ExecForeignBatchInsert only if valid */ -} CopyInsertMethod; - -/* - * This struct contains all the state variables used throughout a COPY FROM - * operation. - */ -typedef struct CopyFromStateData -{ - /* format routine */ - const struct CopyFromRoutine *routine; - - /* low-level state data */ - CopySource copy_src; /* type of copy source */ - FILE *copy_file; /* used if copy_src == COPY_FILE */ - StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ + CopyFromStateData base; EolType eol_type; /* EOL type of input */ int file_encoding; /* file or remote side's character encoding */ bool need_transcoding; /* file encoding diff from server? */ Oid conversion_proc; /* encoding conversion function */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ - Node *whereClause; /* WHERE condition (or NULL) */ - - /* these are just for error messages, see CopyFromErrorCallback */ - const char *cur_relname; /* table name for error messages */ - uint64 cur_lineno; /* line number for error messages */ - const char *cur_attname; /* current att for error messages */ - const char *cur_attval; /* current att value for error messages */ - bool relname_only; /* don't output line number, att, etc. */ - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - AttrNumber num_defaults; /* count of att that are missing and have - * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ - ErrorSaveContext *escontext; /* soft error trapped during in_functions - * execution */ - uint64 num_errors; /* total number of rows which contained soft - * errors */ - int *defmap; /* array of default att numbers related to - * missing att */ - ExprState **defexprs; /* array of default att expressions for all - * att */ bool *defaults; /* if DEFAULT marker was found for * corresponding att */ - bool volatile_defexprs; /* is any of defexprs volatile? */ - List *range_table; /* single element list of RangeTblEntry */ - List *rteperminfos; /* single element list of RTEPermissionInfo */ - ExprState *qualexpr; - - TransitionCaptureState *transition_capture; /* * These variables are used to reduce overhead in COPY FROM. @@ -141,7 +59,6 @@ typedef struct CopyFromStateData * appropriate. (In binary mode, line_buf is not used.) */ StringInfoData line_buf; - bool line_buf_valid; /* contains the row being processed? */ /* * input_buf holds input data, already converted to database encoding. @@ -175,23 +92,22 @@ typedef struct CopyFromStateData char *raw_buf; int raw_buf_index; /* next byte to process */ int raw_buf_len; /* total # of bytes stored */ - bool raw_reached_eof; /* true if we reached EOF */ /* Shorthand for number of unconsumed bytes available in raw_buf */ #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) - - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; +} CopyFromStateBuiltins; extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); /* One-row callbacks for built-in formats defined in copyfromparse.c */ extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); extern bool CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); + +extern Size CopyFromBuiltinsEstimateSpace(void); #endif /* COPYFROM_INTERNAL_H */ diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h index 7561083a323..145dccd0f8f 100644 --- a/src/include/commands/copystate.h +++ b/src/include/commands/copystate.h @@ -17,6 +17,8 @@ #include "postgres.h" #include "commands/copy.h" #include "executor/execdesc.h" +#include "commands/trigger.h" +#include "nodes/miscnodes.h" /* * Represents the different dest cases we need to worry about at @@ -65,4 +67,99 @@ typedef struct CopyToStateData uint64 bytes_processed; /* number of bytes processed so far */ } CopyToStateData; +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_FILE, /* from file (or a piped program) */ + COPY_FRONTEND, /* from frontend */ + COPY_CALLBACK, /* from callback function */ +} CopySource; + +/* + * Represents the end-of-line terminator type of the input + */ +typedef enum EolType +{ + EOL_UNKNOWN, + EOL_NL, + EOL_CR, + EOL_CRNL, +} EolType; + +/* + * Represents the insert method to be used during COPY FROM. + */ +typedef enum CopyInsertMethod +{ + CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */ + CIM_MULTI, /* always use table_multi_insert or + * ExecForeignBatchInsert */ + CIM_MULTI_CONDITIONAL, /* use table_multi_insert or + * ExecForeignBatchInsert only if valid */ +} CopyInsertMethod; + +/* + * This struct contains all the state variables used throughout a COPY FROM + * operation. + */ +typedef struct CopyFromStateData +{ + /* format routine */ + const struct CopyFromRoutine *routine; + + /* low-level state data */ + CopySource copy_src; /* type of copy source */ + FILE *copy_file; /* used if copy_src == COPY_FILE */ + StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ + bool reached_eof; /* true if we reached EOF */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + + /* these are just for error messages, see CopyFromErrorCallback */ + const char *cur_relname; /* table name for error messages */ + uint64 cur_lineno; /* line number for error messages */ + const char *cur_attname; /* current att for error messages */ + const char *cur_attval; /* current att value for error messages */ + bool relname_only; /* don't output line number, att, etc. */ + StringInfo line_buf; + bool line_buf_valid; + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapped during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ + AttrNumber num_defaults; /* count of att that are missing and have + * default value */ + int *defmap; /* array of default att numbers related to + * missing att */ + ExprState **defexprs; /* array of default att expressions for all + * att */ + bool volatile_defexprs; /* is any of defexprs volatile? */ + List *range_table; /* single element list of RangeTblEntry */ + List *rteperminfos; /* single element list of RTEPermissionInfo */ + ExprState *qualexpr; + + TransitionCaptureState *transition_capture; + + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyFromStateData; + #endif -- 2.47.3 [application/octet-stream] 0001-Separate-format-specific-fields-from-CopyToStateData.patch (21.5K, ../../CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com/7-0001-Separate-format-specific-fields-from-CopyToStateData.patch) download | inline diff: From 7d8ac622144e20c9e3aaf958bc0644a3c74fffea Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Thu, 6 Nov 2025 00:02:03 -0800 Subject: [PATCH 1/6] Separate format-specific fields from CopyToStateData struct. Previously, CopyToStateData struct contained all state variables used throughout COPY TO operations. To support the upcoming pluggable COPY format feature, this commit moves fields specific to built-in formats into their own dedicated struct. This commit also introduces a new callback CopyToEstimateStateSpace, allowing format routines to estimate and return the memory size needed for their state data. The format's state data must embed CopyToStateData as its first field, and the returned size must be equal to or larger than CopyToStateData. While separate structs for core and format-specific usage might seem feasible, using contiguousmemory avoids overhead from extra pointer traversal. --- src/backend/commands/copyto.c | 249 ++++++++++++++++--------------- src/include/commands/copy.h | 6 +- src/include/commands/copyapi.h | 5 + src/include/commands/copystate.h | 68 +++++++++ 4 files changed, 208 insertions(+), 120 deletions(-) create mode 100644 src/include/commands/copystate.h diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cef452584e5..6a0a66507ba 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -22,6 +22,8 @@ #include "access/tableam.h" #include "catalog/pg_inherits.h" #include "commands/copyapi.h" +#include "commands/copystate.h" +#include "commands/defrem.h" #include "commands/progress.h" #include "executor/execdesc.h" #include "executor/executor.h" @@ -39,19 +41,7 @@ #include "utils/snapmgr.h" /* - * Represents the different dest cases we need to worry about at - * the bottom level - */ -typedef enum CopyDest -{ - COPY_FILE, /* to file (or a piped program) */ - COPY_FRONTEND, /* to frontend */ - COPY_CALLBACK, /* to callback function */ -} CopyDest; - -/* - * This struct contains all the state variables used throughout a COPY TO - * operation. + * Struct used for text and CSV format. * * Multi-byte encodings: all supported client-side encodings encode multi-byte * characters by having the first byte's high bit set. Subsequent bytes of the @@ -64,41 +54,14 @@ typedef enum CopyDest * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true * when we have to do it the hard way. */ -typedef struct CopyToStateData +typedef struct CopyToStateTextLike { - /* format-specific routines */ - const CopyToRoutine *routine; - - /* low-level state data */ - CopyDest copy_dest; /* type of copy source/destination */ - FILE *copy_file; /* used if copy_dest == COPY_FILE */ - StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + CopyToStateData base; int file_encoding; /* file or remote side's character encoding */ bool need_transcoding; /* file encoding diff from server? */ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - List *partitions; /* OID list of partitions to copy data from */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - FmgrInfo *out_functions; /* lookup info for output functions */ - MemoryContext rowcontext; /* per-row evaluation context */ - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; +} CopyToStateTextLike; /* DestReceiver for COPY (query) TO */ typedef struct @@ -116,13 +79,14 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; static void EndCopy(CopyToState cstate); static void ClosePipeToProgram(CopyToState cstate); static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot); -static void CopyAttributeOutText(CopyToState cstate, const char *string); -static void CopyAttributeOutCSV(CopyToState cstate, const char *string, +static void CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string); +static void CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string, bool use_quote); static void CopyRelationTo(CopyToState cstate, Relation rel, Relation root_rel, uint64 *processed); /* built-in format-specific routines */ +static Size CopyToEstimateStateTextLike(void); static void CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc); static void CopyToTextLikeOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo); static void CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot); @@ -130,6 +94,7 @@ static void CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot); static void CopyToTextLikeOneRow(CopyToState cstate, TupleTableSlot *slot, bool is_csv); static void CopyToTextLikeEnd(CopyToState cstate); +static Size CopyToEstimateStateBinary(void); static void CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc); static void CopyToBinaryOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo); static void CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot); @@ -155,6 +120,7 @@ static void CopySendInt16(CopyToState cstate, int16 val); /* text format */ static const CopyToRoutine CopyToRoutineText = { + .CopyToEstimateStateSpace = CopyToEstimateStateTextLike, .CopyToStart = CopyToTextLikeStart, .CopyToOutFunc = CopyToTextLikeOutFunc, .CopyToOneRow = CopyToTextOneRow, @@ -163,6 +129,7 @@ static const CopyToRoutine CopyToRoutineText = { /* CSV format */ static const CopyToRoutine CopyToRoutineCSV = { + .CopyToEstimateStateSpace = CopyToEstimateStateTextLike, .CopyToStart = CopyToTextLikeStart, .CopyToOutFunc = CopyToTextLikeOutFunc, .CopyToOneRow = CopyToCSVOneRow, @@ -171,62 +138,77 @@ static const CopyToRoutine CopyToRoutineCSV = { /* binary format */ static const CopyToRoutine CopyToRoutineBinary = { + .CopyToEstimateStateSpace = CopyToEstimateStateBinary, .CopyToStart = CopyToBinaryStart, .CopyToOutFunc = CopyToBinaryOutFunc, .CopyToOneRow = CopyToBinaryOneRow, .CopyToEnd = CopyToBinaryEnd, }; -/* Return a COPY TO routine for the given options */ -static const CopyToRoutine * -CopyToGetRoutine(const CopyFormatOptions *opts) +static Size +CopyToEstimateStateTextLike(void) { - if (opts->csv_mode) - return &CopyToRoutineCSV; - else if (opts->binary) - return &CopyToRoutineBinary; - - /* default is text */ - return &CopyToRoutineText; + return sizeof(CopyToStateTextLike); } /* Implementation of the start callback for text and CSV formats */ static void -CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc) +CopyToTextLikeStart(CopyToState ccstate, TupleDesc tupDesc) { + CopyToStateTextLike *cstate = (CopyToStateTextLike *) ccstate; + + /* Use client encoding when ENCODING option is not specified. */ + if (cstate->base.opts.file_encoding < 0) + cstate->file_encoding = pg_get_client_encoding(); + else + cstate->file_encoding = cstate->base.opts.file_encoding; + + /* + * Set up encoding conversion info if the file and server encodings differ + * (see also pg_server_to_any). + */ + if (cstate->file_encoding == GetDatabaseEncoding() || + cstate->file_encoding == PG_SQL_ASCII) + cstate->need_transcoding = false; + else + cstate->need_transcoding = true; + + /* See Multibyte encoding comment above */ + cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); + /* * For non-binary copy, we need to convert null_print to file encoding, * because it will be sent directly with CopySendString. */ if (cstate->need_transcoding) - cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, - cstate->opts.null_print_len, - cstate->file_encoding); + cstate->base.opts.null_print_client = pg_server_to_any(cstate->base.opts.null_print, + cstate->base.opts.null_print_len, + cstate->file_encoding); /* if a header has been requested send the line */ - if (cstate->opts.header_line == COPY_HEADER_TRUE) + if (cstate->base.opts.header_line == COPY_HEADER_TRUE) { ListCell *cur; bool hdr_delim = false; - foreach(cur, cstate->attnumlist) + foreach(cur, cstate->base.attnumlist) { int attnum = lfirst_int(cur); char *colname; if (hdr_delim) - CopySendChar(cstate, cstate->opts.delim[0]); + CopySendChar(ccstate, cstate->base.opts.delim[0]); hdr_delim = true; colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) + if (cstate->base.opts.csv_mode) CopyAttributeOutCSV(cstate, colname, false); else CopyAttributeOutText(cstate, colname); } - CopySendTextLikeEndOfRow(cstate); + CopySendTextLikeEndOfRow(ccstate); } } @@ -294,10 +276,10 @@ CopyToTextLikeOneRow(CopyToState cstate, value); if (is_csv) - CopyAttributeOutCSV(cstate, string, + CopyAttributeOutCSV((CopyToStateTextLike *) cstate, string, cstate->opts.force_quote_flags[attnum - 1]); else - CopyAttributeOutText(cstate, string); + CopyAttributeOutText((CopyToStateTextLike *) cstate, string); } } @@ -311,6 +293,13 @@ CopyToTextLikeEnd(CopyToState cstate) /* Nothing to do here */ } +static Size +CopyToEstimateStateBinary(void) +{ + /* Binary format doesn't require additional fields */ + return sizeof(CopyToStateData); +} + /* * Implementation of the start callback for binary format. Send a header * for a binary copy. @@ -406,7 +395,7 @@ SendCopyBegin(CopyToState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->copy_dest = COPY_DEST_FRONTEND; } static void @@ -453,7 +442,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || ferror(cstate->copy_file)) @@ -487,11 +476,11 @@ CopySendEndOfRow(CopyToState cstate) errmsg("could not write to COPY file: %m"))); } break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; - case COPY_CALLBACK: + case COPY_DEST_CALLBACK: cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len); break; } @@ -512,7 +501,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) { switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: /* Default line termination depends on platform */ #ifndef WIN32 CopySendChar(cstate, '\n'); @@ -520,7 +509,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) CopySendString(cstate, "\r\n"); #endif break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); break; @@ -614,6 +603,54 @@ EndCopy(CopyToState cstate) pfree(cstate); } +/* + * Allocate COPY TO state data based on the format's EsimateStateSpace + * callback. + */ +static CopyToState +create_copyto_state(ParseState *pstate, List *options) +{ + const CopyToRoutine *routine; + CopyToState cstate; + Size req_size; + bool format_specified = false; + + routine = &CopyToRoutineText; /* default */ + foreach_node(DefElem, defel, options) + { + if (strcmp(defel->defname, "format") == 0) + { + char *fmt = defGetString(defel); + + if (format_specified) + errorConflictingDefElem(defel, pstate); + format_specified = true; + if (strcmp(fmt, "text") == 0) + routine = &CopyToRoutineText; + else if (strcmp(fmt, "csv") == 0) + routine = &CopyToRoutineCSV; + else if (strcmp(fmt, "binary") == 0) + routine = &CopyToRoutineBinary; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", fmt), + parser_errposition(pstate, defel->location))); + + break; + } + } + + req_size = routine->CopyToEstimateStateSpace(); + Assert(req_size >= sizeof(CopyToStateData)); + + /* Allocate workspace and zero all fields */ + cstate = (CopyToState) palloc0(req_size); + cstate->routine = routine; + + return cstate; +} + /* * Setup CopyToState to read tuples from a table or a query for COPY TO. * @@ -718,9 +755,7 @@ BeginCopyTo(ParseState *pstate, RelationGetRelationName(rel)))); } - - /* Allocate workspace and zero all fields */ - cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateData)); + cstate = create_copyto_state(pstate, options); /* * We allocate everything used by a cstate in a new memory context. This @@ -735,9 +770,6 @@ BeginCopyTo(ParseState *pstate, /* Extract options from the statement node tree */ ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options); - /* Set format routine */ - cstate->routine = CopyToGetRoutine(&cstate->opts); - /* Process the source/target relation or query */ if (rel) { @@ -918,31 +950,12 @@ BeginCopyTo(ParseState *pstate, } } - /* Use client encoding when ENCODING option is not specified. */ - if (cstate->opts.file_encoding < 0) - cstate->file_encoding = pg_get_client_encoding(); - else - cstate->file_encoding = cstate->opts.file_encoding; - - /* - * Set up encoding conversion info if the file and server encodings differ - * (see also pg_server_to_any). - */ - if (cstate->file_encoding == GetDatabaseEncoding() || - cstate->file_encoding == PG_SQL_ASCII) - cstate->need_transcoding = false; - else - cstate->need_transcoding = true; - - /* See Multibyte encoding comment above */ - cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - - cstate->copy_dest = COPY_FILE; /* default */ + cstate->copy_dest = COPY_DEST_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->copy_dest = COPY_DEST_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) @@ -1233,16 +1246,16 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) #define DUMPSOFAR() \ do { \ if (ptr > start) \ - CopySendData(cstate, start, ptr - start); \ + CopySendData((CopyToState) cstate, start, ptr - start); \ } while (0) static void -CopyAttributeOutText(CopyToState cstate, const char *string) +CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string) { const char *ptr; const char *start; char c; - char delimc = cstate->opts.delim[0]; + char delimc = cstate->base.opts.delim[0]; if (cstate->need_transcoding) ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); @@ -1307,14 +1320,14 @@ CopyAttributeOutText(CopyToState cstate, const char *string) } /* if we get here, we need to convert the control char */ DUMPSOFAR(); - CopySendChar(cstate, '\\'); - CopySendChar(cstate, c); + CopySendChar((CopyToState) cstate, '\\'); + CopySendChar((CopyToState) cstate, c); start = ++ptr; /* do not include char in next run */ } else if (c == '\\' || c == delimc) { DUMPSOFAR(); - CopySendChar(cstate, '\\'); + CopySendChar((CopyToState) cstate, '\\'); start = ptr++; /* we include char in next run */ } else if (IS_HIGHBIT_SET(c)) @@ -1367,14 +1380,14 @@ CopyAttributeOutText(CopyToState cstate, const char *string) } /* if we get here, we need to convert the control char */ DUMPSOFAR(); - CopySendChar(cstate, '\\'); - CopySendChar(cstate, c); + CopySendChar((CopyToState) cstate, '\\'); + CopySendChar((CopyToState) cstate, c); start = ++ptr; /* do not include char in next run */ } else if (c == '\\' || c == delimc) { DUMPSOFAR(); - CopySendChar(cstate, '\\'); + CopySendChar((CopyToState) cstate, '\\'); start = ptr++; /* we include char in next run */ } else @@ -1390,19 +1403,19 @@ CopyAttributeOutText(CopyToState cstate, const char *string) * CSV-style escaping */ static void -CopyAttributeOutCSV(CopyToState cstate, const char *string, +CopyAttributeOutCSV(CopyToStateTextLike * cstate, const char *string, bool use_quote) { const char *ptr; const char *start; char c; - char delimc = cstate->opts.delim[0]; - char quotec = cstate->opts.quote[0]; - char escapec = cstate->opts.escape[0]; - bool single_attr = (list_length(cstate->attnumlist) == 1); + char delimc = cstate->base.opts.delim[0]; + char quotec = cstate->base.opts.quote[0]; + char escapec = cstate->base.opts.escape[0]; + bool single_attr = (list_length(cstate->base.attnumlist) == 1); /* force quoting if it matches null_print (before conversion!) */ - if (!use_quote && strcmp(string, cstate->opts.null_print) == 0) + if (!use_quote && strcmp(string, cstate->base.opts.null_print) == 0) use_quote = true; if (cstate->need_transcoding) @@ -1445,7 +1458,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, if (use_quote) { - CopySendChar(cstate, quotec); + CopySendChar((CopyToState) cstate, quotec); /* * We adopt the same optimization strategy as in CopyAttributeOutText @@ -1456,7 +1469,7 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, if (c == quotec || c == escapec) { DUMPSOFAR(); - CopySendChar(cstate, escapec); + CopySendChar((CopyToState) cstate, escapec); start = ptr; /* we include char in next run */ } if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) @@ -1466,12 +1479,12 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, } DUMPSOFAR(); - CopySendChar(cstate, quotec); + CopySendChar((CopyToState) cstate, quotec); } else { /* If it doesn't need quoting, we can just dump it as-is */ - CopySendString(cstate, ptr); + CopySendString((CopyToState) cstate, ptr); } } diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 541176e1980..d75a70715a4 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -87,10 +87,12 @@ typedef struct CopyFormatOptions List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; -/* These are private in commands/copy[from|to].c */ -typedef struct CopyFromStateData *CopyFromState; +/* defined in copystate.h */ typedef struct CopyToStateData *CopyToState; +/* Private in commands/copyfrom.c */ +typedef struct CopyFromStateData *CopyFromState; + typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); typedef void (*copy_data_dest_cb) (void *data, int len); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 2a2d2f9876b..3b9982d54b8 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -22,6 +22,11 @@ */ typedef struct CopyToRoutine { + /* + * Estimate and return the memory size required to store the state data. + */ + Size (*CopyToEstimateStateSpace) (void); + /* * Set output function information. This callback is called once at the * beginning of COPY TO. diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h new file mode 100644 index 00000000000..7561083a323 --- /dev/null +++ b/src/include/commands/copystate.h @@ -0,0 +1,68 @@ +/*------------------------------------------------------------------------- + * + * copystate.h + * Definitions for state data used by COPY TO and COPY FROM. + * + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/commands/copystate.h + * + *------------------------------------------------------------------------- + */ +#ifndef COPYSTATE_H +#define COPYSTATE_H + +#include "postgres.h" +#include "commands/copy.h" +#include "executor/execdesc.h" + +/* + * Represents the different dest cases we need to worry about at + * the bottom level + */ +typedef enum CopyDest +{ + COPY_DEST_FILE, /* to/from file (or a piped program) */ + COPY_DEST_FRONTEND, /* to/from frontend */ + COPY_DEST_CALLBACK, /* to/from callback function */ +} CopyDest; + +/* + * This struct contains the state variables commonly used throughout a + * COPY TO operation. + */ +typedef struct CopyToStateData +{ + /* format-specific routines */ + const struct CopyToRoutine *routine; + + /* low-level state data */ + CopyDest copy_dest; /* type of copy source/destination */ + FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + List *partitions; /* OID list of partitions to copy data from */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *out_functions; /* lookup info for output functions */ + MemoryContext rowcontext; /* per-row evaluation context */ + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyToStateData; + +#endif -- 2.47.3 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-11-17 06:40 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2025-11-17 06:40 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoDCEfe0PQhMEx8G1rpS7RrzGCJPobeqm3Mpn2bgbUH9nQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 14 Nov 2025 12:19:47 -0800, Masahiko Sawada <[email protected]> wrote: > This thread has involved extensive discussion, and the patch needs to > be rebased. I'd like to summarize the current status of this patch and > our discussions. I've attached updated patches that implement the > whole ideas of this feature to help provide a clearer overall picture. Thanks! I'm not sure whether we should include option parsing feature to this patch's scope or not but I'm OK with this approach. Here are my review comments but they are minor comments. They don't require design change. 0001: diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cef452584e5..6a0a66507ba 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -155,6 +120,7 @@ static void CopySendInt16(CopyToState cstate, int16 val); /* text format */ static const CopyToRoutine CopyToRoutineText = { + .CopyToEstimateStateSpace = CopyToEstimateStateTextLike, How about including "Space" (CopyToEstimateStateSpaceTextLike)? How about renaming this to "CopyToTextLikeEstimateStateSpace" because other functions use "CopyToTextLikeXXX" style. @@ -171,62 +138,77 @@ static const CopyToRoutine CopyToRoutineCSV = { ... /* Implementation of the start callback for text and CSV formats */ static void -CopyToTextLikeStart(CopyToState cstate, TupleDesc tupDesc) +CopyToTextLikeStart(CopyToState ccstate, TupleDesc tupDesc) { + CopyToStateTextLike *cstate = (CopyToStateTextLike *) ccstate; How about always using "cstate" for "CopyToState"? In other functions in this patch, "CopyToState" is referred "cstate" or "cctate". We can use "state" or something for "CopyToStateTextLike". (0002 uses "state" for "CopyFromStateBuiltins".) + cstate->base.opts.null_print_client = pg_server_to_any(cstate->base.opts.null_print, + cstate->base.opts.null_print_len, + cstate->file_encoding); We can use "ccstate->" instead of "cstate->base." in this function. @@ -614,6 +603,54 @@ EndCopy(CopyToState cstate) pfree(cstate); } +/* + * Allocate COPY TO state data based on the format's EsimateStateSpace + * callback. + */ +static CopyToState +create_copyto_state(ParseState *pstate, List *options) "Esimate" -> "Estimate" How about using CamelCase like "CreateCopyToState" for function name like other functions in this file? + Size req_size; "state_size" may be better. @@ -1233,16 +1246,16 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) ... static void -CopyAttributeOutText(CopyToState cstate, const char *string) +CopyAttributeOutText(CopyToStateTextLike * cstate, const char *string) { CopyAttributeOutText(CopyToState cstate, const char *string) { CopyToStateTextLike *state = (CopyToStateTextLike *)cstate; may reduce "cstate->base." and "(CopyToState) cstate" in this function. BTW, could you remove a needless space after "*" in "CopyToStateTextLike * cstate"? 0002: diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 12781963b4f..0c51e5ba5e1 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -129,6 +131,7 @@ static void CopyFromBinaryEnd(CopyFromState cstate); /* text format */ static const CopyFromRoutine CopyFromRoutineText = { + .CopyFromEstimateStateSpace = CopyFromBuiltinsEstimateSpace, How about including "State" ("CopyFromBuiltinsEstimateStateSpace")? @@ -145,54 +149,129 @@ static const CopyFromRoutine CopyFromRoutineCSV = { ... +/* + * Common routine to initialize CopyFromStateBuiltins data. + */ +static void +initialize_copyfrom_bultins_state(CopyFromStateBuiltins * state, TupleDesc tupDesc) * How about using CamelCase like other functions in this file? * How about using the same name as the struct? InitializeCopyFromStateBuiltins? @@ -1379,8 +1462,8 @@ CopyFrom(CopyFromState cstate) /* Add this tuple to the tuple buffer */ CopyMultiInsertInfoStore(&multiInsertInfo, resultRelInfo, myslot, - cstate->line_buf.len, - cstate->cur_lineno); + rowinfo.tuplen, + rowinfo.lineno); How about passing "CopyFromRowInfo *" instead of "rowinfo.tuplen" and "rowinfo.lineno"? @@ -1512,6 +1595,50 @@ CopyFrom(CopyFromState cstate) return processed; } +static CopyFromState +create_copyfrom_state(ParseState *pstate, List *options) How about using CamelCase like other functions in this file? CreateCopyFromState? @@ -1138,19 +1156,29 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, ... + if (rowinfo) + { + /* + * XXX: We used to use line_buf.len but we don't actually use line_buf + * in binary format. + */ + rowinfo->lineno = cstate->base.cur_lineno; + rowinfo->tuplen = cstate->line_buf.len; } How about always setting "0" to "rowinfo->tuplen" instead of using "cstate->line_buf.len"? diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index d75a70715a4..30a1d2bff6e 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -48,6 +48,12 @@ typedef enum CopyLogVerbosityChoice ... +typedef struct CopyFromRowInfo +{ + uint64 lineno; + int tuplen; +} CopyFromRowInfo; I don't have a strong opinion nor alternative but I'm not sure whether this name is suitable or not... diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 3b9982d54b8..c3d2199a0b6 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -99,12 +104,11 @@ typedef struct CopyFromRoutine * Returns false if there are no more tuples to read. */ bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls); + Datum *values, bool *nulls, CopyFromRowInfo * rowinfo); Can we add some docstrings for "rowinfo"? diff --git a/src/include/commands/copystate.h b/src/include/commands/copystate.h index 7561083a323..145dccd0f8f 100644 --- a/src/include/commands/copystate.h +++ b/src/include/commands/copystate.h @@ -65,4 +67,99 @@ typedef struct CopyToStateData ... +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_FILE, /* from file (or a piped program) */ + COPY_FRONTEND, /* from frontend */ + COPY_CALLBACK, /* from callback function */ +} CopySource; Can we use "COPY_SOURCE_" prefix instead of "COPY_" prefix such as "COPY_SOURCE_FILE"? 0003: diff --git a/src/backend/commands/copy_custom_format.c b/src/backend/commands/copy_custom_format.c new file mode 100644 index 00000000000..8bef6e779ac --- /dev/null +++ b/src/backend/commands/copy_custom_format.c +/* + * Returns true if the given custom format name is registered. + */ +bool +FindCustomCopyFormat(const char *fmt_name) +{ + for (int i = 0; i < CopyCustomFormatAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0) + return true; + } + + return false; +} How about using other word than "Find" here? I expect that "FindXXX()" returns a found value instead of "whether found or not" as bool. CustomCopyFormatExists()? +bool +GetCustomCopyToRoutine(const char *fmt_name, const CopyToRoutine **to_routine) +{ + for (int i = 0; i < CopyCustomFormatAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].fmt_name, fmt_name) == 0) + { + *to_routine = CopyCustomFormatArray[i].to_routine; + return true; + } + } + + return false; +} How about returning "const CopyToRoutine *" instead of "bool"? We can use "FindCustomCopyFormat()" whether the given format name exists or not. +bool +GetCustomCopyFromRoutine(const char *fmt_name, const CopyFromRoutine **from_routine) Ditto. diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 30a1d2bff6e..82f07b05823 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -120,6 +120,12 @@ extern uint64 CopyFrom(CopyFromState cstate); extern DestReceiver *CreateCopyDestReceiver(void); +extern void ProcessCopyBuiltinOptions(List *options, CopyFormatOptions *opts_out, + bool is_from, List **other_options, ParseState *pstate); It seems that this change should exist in 0004. 0004: diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b1b3ae141eb..ea31fa911f9 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -1570,3 +1571,32 @@ CreateCopyDestReceiver(void) return (DestReceiver *) self; } + +static void +ProcessCopyToOptions(CopyToState cstate, List *options, ParseState *pstate) +{ + bool temp_state = false; + List *other_options = NIL; + CopyFormatOptions *opts; + + if (cstate == NULL) + { + cstate = create_copyto_state(pstate, options); + temp_state = true; + } + + opts = &cstate->opts; + + ProcessCopyBuiltinOptions(options, opts, false, &other_options, pstate); + + foreach_node(DefElem, option, options) options -> other_options (This change exists in 0005.) 0006: diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c new file mode 100644 index 00000000000..a63390e875b --- /dev/null +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c +static Size +TestCopyToEsimateSpace(void) +{ + return sizeof(TestCopyToState); +} + +static Size +TestCopyFromEsimateSpace(void) +{ + return sizeof(TestCopyFromState); +} EsimateSpace -> EstimateStateSpace + if (strcmp(option->defname, "common_int") == 0) + { + int val = defGetInt32(option); + + opt->common_int = val; + + return true; + } + else if (strcmp(option->defname, "common_bool") == 0) + { + bool val = defGetBoolean(option); + + opt->common_bool = val; + + return true; We may not need to use local variables: opt->common_int = defGetInt32(option); opt->common_bool = defGetBoolean(option); > One potential improvement would be adding support for random file > access in COPY FROM operations. For example, with parquet files, it > would be much more efficient to read the footer section first since it > contains metadata, allowing selective reading of necessary file > sections. The current sequential read API (CopyFromGetData()) requires > reading all data to access the metadata. This is outside the scope of this patch but I've created a custom COPY format implementation for Apache Parquet: https://github.com/kou/pg-copy-parquet We need to start parsing from footer as mentioned above. So the implementation reads all data before it starts parsing: https://github.com/kou/pg-copy-parquet/blob/7da367ea81d8964f5045fe0b1514a798d4ecbbc7/copy_parquet.cc... If we have random access API, we don't need to read all data. It improves performance. Anyway, this is outside the scope of this patch. We can discuss this in a separated thread after we merge this patch. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-11-17 17:04 Tomas Vondra <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 127+ messages in thread From: Tomas Vondra @ 2025-11-17 17:04 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; Sutou Kouhei <[email protected]>; [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On 11/14/25 21:19, Masahiko Sawada wrote: > After offline discussions with Sutou-san, we believe the current APIs > work well, particularly for text-based formats, though we still need > to verify there are no performance regressions. I got pinged about this patch off-list. I won't have capacity to do a proper review, anytime soon, but I got a bit of time to do a simple benchmark (which seems useful as that was one of the concerns in this thread, it seems). Attached is a script that does COPY TO/FROM with the built-in formats, on table with 1, 10 and 100 integer columns. The data sets are between 10 and 1M rows. The table is UNLOGGED, to eliminate WAL overhead. The attached PDF summarizes results from my ryzen machine, for master and patched build. The final columns are comparison (i.e. copy/master) of the timings. Values >100% are regressions (marked as red). It seems quite "red", but it's not particularly conclusive. The differences are mostly within 5%, and that could be caused e.g. by changes to binary layout. And some of the cases got faster too. It might be interesting to get results from other machines. The script may need some adjustments, but it should be too difficult. The other thing Andres was concerned about is "the amount of COPY performance improvements it forecloses". I have no opinion on that, as it depends on what improvements Andres envisioned. regards -- Tomas Vondra Attachments: [application/x-shellscript] copy.sh (1.9K, ../../[email protected]/2-copy.sh) download [application/pdf] copy.pdf (46.4K, ../../[email protected]/3-copy.pdf) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-12-02 02:39 Sutou Kouhei <[email protected]> parent: Tomas Vondra <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2025-12-02 02:39 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Nov 2025 18:04:46 +0100, Tomas Vondra <[email protected]> wrote: > I got pinged about this patch off-list. I won't have capacity to do a > proper review, anytime soon, but I got a bit of time to do a simple > benchmark (which seems useful as that was one of the concerns in this > thread, it seems). Thanks!!! I also do the same condition benchmark on my Mac mini: Machine: * Apple M1 (8 core) * Memory: 16GB * macOS Sequoia (15.6) Parameters: * N integer columns: 1 10 100 * N rows: 10 100 1000 10000 10000 100000 1000000 * Formats: text csv binary * Operations: FROM TO * Patches: 0001 0002 0003 0004 0005 0006 I ran 5 times for each parameter set and choose the median elapsed time. I used https://gitlab.com/ktou/pg-bench/-/blob/main/copy-format-extendable/run.sh . I attach it. I measured 6 times. See the attached mac-mini-result-${N}.{csv,pdf}. A PDF has 3 columns. They are text, csv and binary formats from left to right. 1st row uses COPY FROM and 0001 patch. 2nd row uses COPY TO and 0001 patch. 3rd row uses COPY FROM and 0002 patch. 4th row uses COPY TO and 0002 patch. ... Each heatmap visualizes (${elapsed_time_patch} / ${elapsed_time_master}) * 100. 100 > (red) means slower and 100 < (blue) means faster. It seems that they don't show any reproducible trends. For example, the binary cases in mac-mini-result-2.pdf show that patched cases are always slower but the binary cases in mac-mini-result-{1,6}.pdf show that most patched cases are faster. The binary cases in mac-mini-result-{1,5,6}.pdf show that 0006 patch is slower than 0005 patch but the binary cases in mac-mini-result-{3,4}.pdf don't show it. Another example, the text cases in mac-mini-result-1.pdf show that patched cases are always slower but the text cases in mac-mini-result-1.pdf show that most patched cases are faster. I hope that these numbers help to proceed this proposal. Thanks, -- kou #!/bin/bash # -*- sh-indentation: 2; sh-basic-offset: 2-*- # This is based on codes from Michael Paquier and Tomas Vondra: # * https://www.postgresql.org/message-id/flat/Zbr6piWuVHDtFFOl%40paquier.xyz#dbbec4d5c54ef2317be01a54ab... # * https://www.postgresql.org/message-id/flat/26541788-8853-4d93-86cd-5f701b13ae51%40enterprisedb.com DIR=${1:-$(pwd)} psql postgres > /dev/null 2>&1 <<EOF DROP DATABASE IF EXISTS test; CREATE DATABASE test; EOF psql test > /dev/null 2>&1 <<EOF CREATE EXTENSION blackhole_am; CREATE OR REPLACE FUNCTION create_table_cols(tabname text, num_cols int) RETURNS VOID AS \$func\$ DECLARE query text; BEGIN query := 'CREATE UNLOGGED TABLE ' || tabname || ' ('; FOR i IN 1..num_cols LOOP query := query || 'a_' || i::text || ' int2 default 1'; IF i != num_cols THEN query := query || ', '; END IF; END LOOP; query := query || ')'; EXECUTE format(query); END \$func\$ LANGUAGE plpgsql; EOF for format in text csv binary; do for n_columns in 1 10 100; do for n_rows in 10 100 1000 10000 10000 100000 1000000; do psql test > /dev/null 2>&1 <<EOF DROP TABLE IF EXISTS to_table; DROP TABLE IF EXISTS from_table; SELECT create_table_cols ('to_table', ${n_columns}); SELECT create_table_cols ('from_table', ${n_columns}); INSERT INTO to_table SELECT 1 FROM generate_series(1, ${n_rows}); COPY to_table TO '$DIR/test.data' WITH (FORMAT ${format}); ALTER TABLE from_table SET ACCESS METHOD blackhole_am; EOF # run COPY TO 5x for r in $(seq 1 5); do s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())") psql test -c "COPY to_table TO '/dev/null' WITH (FORMAT ${format})" > /dev/null 2>&1 d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)") echo "TO,${format},${n_columns},${n_rows},${r},${d}" done # run COPY FROM 5x for r in $(seq 1 5); do s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())") psql test -c "COPY from_table FROM '$DIR/test.data' WITH (FORMAT ${format})" > /dev/null 2>&1 d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)") echo "FROM,${format},${n_columns},${n_rows},${r},${d}" done done done done Attachments: [text/plain] run.sh (2.2K, ../../[email protected]/2-run.sh) download | inline: #!/bin/bash # -*- sh-indentation: 2; sh-basic-offset: 2-*- # This is based on codes from Michael Paquier and Tomas Vondra: # * https://www.postgresql.org/message-id/flat/Zbr6piWuVHDtFFOl%40paquier.xyz#dbbec4d5c54ef2317be01a54abaf495c # * https://www.postgresql.org/message-id/flat/26541788-8853-4d93-86cd-5f701b13ae51%40enterprisedb.com DIR=${1:-$(pwd)} psql postgres > /dev/null 2>&1 <<EOF DROP DATABASE IF EXISTS test; CREATE DATABASE test; EOF psql test > /dev/null 2>&1 <<EOF CREATE EXTENSION blackhole_am; CREATE OR REPLACE FUNCTION create_table_cols(tabname text, num_cols int) RETURNS VOID AS \$func\$ DECLARE query text; BEGIN query := 'CREATE UNLOGGED TABLE ' || tabname || ' ('; FOR i IN 1..num_cols LOOP query := query || 'a_' || i::text || ' int2 default 1'; IF i != num_cols THEN query := query || ', '; END IF; END LOOP; query := query || ')'; EXECUTE format(query); END \$func\$ LANGUAGE plpgsql; EOF for format in text csv binary; do for n_columns in 1 10 100; do for n_rows in 10 100 1000 10000 10000 100000 1000000; do psql test > /dev/null 2>&1 <<EOF DROP TABLE IF EXISTS to_table; DROP TABLE IF EXISTS from_table; SELECT create_table_cols ('to_table', ${n_columns}); SELECT create_table_cols ('from_table', ${n_columns}); INSERT INTO to_table SELECT 1 FROM generate_series(1, ${n_rows}); COPY to_table TO '$DIR/test.data' WITH (FORMAT ${format}); ALTER TABLE from_table SET ACCESS METHOD blackhole_am; EOF # run COPY TO 5x for r in $(seq 1 5); do s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())") psql test -c "COPY to_table TO '/dev/null' WITH (FORMAT ${format})" > /dev/null 2>&1 d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)") echo "TO,${format},${n_columns},${n_rows},${r},${d}" done # run COPY FROM 5x for r in $(seq 1 5); do s=$(psql test -t -A -c "SELECT EXTRACT(EPOCH FROM now())") psql test -c "COPY from_table FROM '$DIR/test.data' WITH (FORMAT ${format})" > /dev/null 2>&1 d=$(psql test -t -A -c "SELECT 1000 * (EXTRACT(EPOCH FROM now()) - $s)") echo "FROM,${format},${n_columns},${n_rows},${r},${d}" done done done done [text/csv] mac-mini-result-1.csv (67.2K, ../../[email protected]/3-mac-mini-result-1.csv) download [application/pdf] mac-mini-result-1.pdf (823.5K, ../../[email protected]/4-mac-mini-result-1.pdf) download [text/csv] mac-mini-result-2.csv (66.6K, ../../[email protected]/5-mac-mini-result-2.csv) download [application/pdf] mac-mini-result-2.pdf (814.9K, ../../[email protected]/6-mac-mini-result-2.pdf) download [text/csv] mac-mini-result-3.csv (67.1K, ../../[email protected]/7-mac-mini-result-3.csv) download [application/pdf] mac-mini-result-3.pdf (823.8K, ../../[email protected]/8-mac-mini-result-3.pdf) download [text/csv] mac-mini-result-4.csv (66.7K, ../../[email protected]/9-mac-mini-result-4.csv) download [application/pdf] mac-mini-result-4.pdf (824.2K, ../../[email protected]/10-mac-mini-result-4.pdf) download [text/csv] mac-mini-result-5.csv (67.2K, ../../[email protected]/11-mac-mini-result-5.csv) download [application/pdf] mac-mini-result-5.pdf (823.4K, ../../[email protected]/12-mac-mini-result-5.pdf) download [text/csv] mac-mini-result-6.csv (67.2K, ../../[email protected]/13-mac-mini-result-6.csv) download [application/pdf] mac-mini-result-6.pdf (823.9K, ../../[email protected]/14-mac-mini-result-6.pdf) download ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2025-12-18 23:43 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2025-12-18 23:43 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Dec 1, 2025 at 6:41 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 17 Nov 2025 18:04:46 +0100, > Tomas Vondra <[email protected]> wrote: > > > I got pinged about this patch off-list. I won't have capacity to do a > > proper review, anytime soon, but I got a bit of time to do a simple > > benchmark (which seems useful as that was one of the concerns in this > > thread, it seems). > > Thanks!!! > > I also do the same condition benchmark on my Mac mini: > > Machine: > > * Apple M1 (8 core) > * Memory: 16GB > * macOS Sequoia (15.6) > > Parameters: > > * N integer columns: 1 10 100 > * N rows: 10 100 1000 10000 10000 100000 1000000 > * Formats: text csv binary > * Operations: FROM TO > * Patches: 0001 0002 0003 0004 0005 0006 > > I ran 5 times for each parameter set and choose the median > elapsed time. > > I used > https://gitlab.com/ktou/pg-bench/-/blob/main/copy-format-extendable/run.sh > . I attach it. > > I measured 6 times. See the attached mac-mini-result-${N}.{csv,pdf}. > > A PDF has 3 columns. They are text, csv and binary formats > from left to right. > > 1st row uses COPY FROM and 0001 patch. > 2nd row uses COPY TO and 0001 patch. > > 3rd row uses COPY FROM and 0002 patch. > 4th row uses COPY TO and 0002 patch. > > ... > > Each heatmap visualizes (${elapsed_time_patch} / > ${elapsed_time_master}) * 100. 100 > (red) means slower and > 100 < (blue) means faster. > > It seems that they don't show any reproducible trends. > > For example, the binary cases in mac-mini-result-2.pdf show > that patched cases are always slower but the binary cases in > mac-mini-result-{1,6}.pdf show that most patched cases are > faster. The binary cases in mac-mini-result-{1,5,6}.pdf show > that 0006 patch is slower than 0005 patch but the binary > cases in mac-mini-result-{3,4}.pdf don't show it. > > Another example, the text cases in mac-mini-result-1.pdf > show that patched cases are always slower but the text cases > in mac-mini-result-1.pdf show that most patched cases are > faster. > > I hope that these numbers help to proceed this proposal. Thank you for sharing the performance test results! I'll run the same benchmark tests on my environment. Looking at these results, it seems that 0001-from-binary cases and 0006-to-binary cases are slower throughout the six results? Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2026-06-23 01:06 Masahiko Sawada <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2026-06-23 01:06 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, On Thu, Mar 26, 2026 at 6:36 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoCLxUhQ0uBjDKXvCEtJBCfF13Ru_7u-Qrrsu+0PPUqcPQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 18 Dec 2025 15:43:07 -0800, > Masahiko Sawada <[email protected]> wrote: > > > Looking at these results, it seems that 0001-from-binary cases and > > 0006-to-binary cases are slower throughout the six results? > > Good point. I didn't notice them. But I feel that it's not > related to the patch set. Because 0001 doesn't change COPY > FROM related code. 0001 just changes COPY TO related > code. And 0006 just adds tests. 0006 doesn't change > implementations. > > > BTW, how to proceed this proposal? It seems that we can't > proceed this proposal without PostgreSQL committers' > attentions but it seems that it's difficult. Sorry for going quiet on this for a while -- I haven't had time to work on it until now. After more thought, I'd like to keep the custom-format changes to the bare minimum and not disturb the existing built-in format processing. In particular, I've dropped the earlier rework that split CopyToStateData / CopyFromStateData to hide built-in-specific fields from extensions. That was my own idea, but I no longer think it pays off: the fields it hid (raw_buf, line_buf, the input buffers, etc.) are only ever used by the built-in text/CSV/binary parsers, and a custom format never touches them -- so visible or not, nothing depends on them, while splitting the struct is invasive to the existing format processing. Touching the Copy state structs is fine in itself; it's the hiding that wasn't worth the cost. Instead, each state struct just gets one opaque pointer for a custom format to keep its own state, and the existing code paths are left alone. Updated patches attached: - 0001 moves CopyFromStateData and CopyToStateData to a new copy_state.h, so extensions can implement their routines without including the *_internal.h headers. It also drops file_fdw.c's dependency on copyfrom_internal.h. - 0002 introduces the registration API and the opaque per-format pointer in both structs. - 0003 adds a callback to validate the COPY options as a whole, called after all options are processed. - 0004 adds the regression tests. I'd like to proceed in this direction barring objections. Feedback is very welcome. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [text/x-patch] v2-0002-Allow-extensions-to-register-custom-format-to-COP.patch (19.1K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/2-v2-0002-Allow-extensions-to-register-custom-format-to-COP.patch) download | inline diff: From 1684e2394e5557f94c17e39b94351a76e601e00d Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 09:21:51 -0700 Subject: [PATCH v2 2/4] Allow extensions to register custom format to COPY TO and COPY FROM. Author: Reviewed-by: Discussion: https://postgr.es/m/ --- src/backend/commands/Makefile | 1 + src/backend/commands/copy.c | 97 ++++++++++++++++++++-- src/backend/commands/copyapi.c | 131 ++++++++++++++++++++++++++++++ src/backend/commands/copyfrom.c | 4 +- src/backend/commands/copyto.c | 4 +- src/backend/commands/meson.build | 1 + src/include/commands/copy.h | 19 +++++ src/include/commands/copy_state.h | 6 ++ src/include/commands/copyapi.h | 37 +++++++++ src/tools/pgindent/typedefs.list | 1 + 10 files changed, 290 insertions(+), 11 deletions(-) create mode 100644 src/backend/commands/copyapi.c diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 5b9d084977e..17b7aa08b55 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -23,6 +23,7 @@ OBJS = \ constraint.o \ conversioncmds.o \ copy.o \ + copyapi.o \ copyfrom.o \ copyfromparse.o \ copyto.o \ diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 003b70852bb..2fdba026ee0 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -23,6 +23,7 @@ #include "access/xact.h" #include "catalog/pg_authid.h" #include "commands/copy.h" +#include "commands/copyapi.h" #include "commands/defrem.h" #include "executor/executor.h" #include "mb/pg_wchar.h" @@ -592,6 +593,14 @@ ProcessCopyOptions(ParseState *pstate, bool force_array_specified = false; ListCell *option; + /* + * Options not recognized by core are collected here and, once the format + * is known, either handed to a custom format's option parser or rejected. + */ + List *deferred_options = NIL; + ProcessOneOptionFn custom_process_option_fn = NULL; + char *custom_format_name = NULL; + /* Support external use for option sanity checking */ if (opts_out == NULL) opts_out = palloc0_object(CopyFormatOptions); @@ -620,6 +629,13 @@ ProcessCopyOptions(ParseState *pstate, opts_out->format = COPY_FORMAT_BINARY; else if (strcmp(fmt, "json") == 0) opts_out->format = COPY_FORMAT_JSON; + else if (GetCopyCustomFormatRoutines(fmt, &opts_out->to_routine, + &opts_out->from_routine, + &custom_process_option_fn)) + { + opts_out->format = COPY_FORMAT_CUSTOM; + custom_format_name = fmt; + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -775,11 +791,54 @@ ProcessCopyOptions(ParseState *pstate, opts_out->reject_limit = defGetCopyRejectLimitOption(defel); } else + { + /* + * Not a core option. Defer the check to after the loop as it may + * belong to a custom format whose "format" option has not been + * seen yet. + */ + deferred_options = lappend(deferred_options, defel); + } + } + + /* + * Now that the format and every option have been seen, resolve the + * deferred options. + */ + if (deferred_options != NIL) + { + /* + * For a custom format, they belong to the handler; for any built-in + * (including the default) an unrecognized option is an error, + * preserving the historical behavior relied on by external callers + * such as file_fdw. + */ + if (opts_out->format != COPY_FORMAT_CUSTOM || custom_process_option_fn == NULL) + { + DefElem *defel = linitial_node(DefElem, deferred_options); + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("option \"%s\" not recognized", defel->defname), parser_errposition(pstate, defel->location))); + } + + /* + * Hand each option core did not recognize to the format's per-option + * callback. Anything the format does not claim (or any option at all + * if it has no callback) is an error, so an unrecognized option + * always fails here. + */ + foreach_node(DefElem, opt, deferred_options) + { + if (!custom_process_option_fn(opts_out, is_from, opt)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("COPY format \"%s\" does not accept option \"%s\"", + custom_format_name, opt->defname), + parser_errposition(pstate, opt->location))); + } } /* @@ -869,7 +928,7 @@ ProcessCopyOptions(ParseState *pstate, * future-proofing. Likewise we disallow all digits though only octal * digits are actually dangerous. */ - if (opts_out->format != COPY_FORMAT_CSV && + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", opts_out->delim[0]) != NULL) ereport(ERROR, @@ -888,7 +947,8 @@ ProcessCopyOptions(ParseState *pstate, : errmsg("cannot specify %s in JSON mode", "HEADER")); /* Check quote */ - if (opts_out->format != COPY_FORMAT_CSV && opts_out->quote != NULL) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + opts_out->quote != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -905,7 +965,8 @@ ProcessCopyOptions(ParseState *pstate, errmsg("COPY delimiter and quote must be different"))); /* Check escape */ - if (opts_out->format != COPY_FORMAT_CSV && opts_out->escape != NULL) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + opts_out->escape != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -917,7 +978,8 @@ ProcessCopyOptions(ParseState *pstate, errmsg("COPY escape must be a single one-byte character"))); /* Check force_quote */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_quote || opts_out->force_quote_all)) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_quote || opts_out->force_quote_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -931,8 +993,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY FROM"))); /* Check force_notnull */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_notnull != NIL || - opts_out->force_notnull_all)) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_notnull != NIL || opts_out->force_notnull_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -947,8 +1009,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY TO"))); /* Check force_null */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_null != NIL || - opts_out->force_null_all)) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_null != NIL || opts_out->force_null_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -995,7 +1057,8 @@ ProcessCopyOptions(ParseState *pstate, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY %s is not supported for %s", "FORMAT JSON", "COPY FROM")); - if (opts_out->format != COPY_FORMAT_JSON && opts_out->force_array) + if (CopyFormatBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_JSON && + opts_out->force_array) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY %s can only be used with JSON mode", "FORCE_ARRAY")); @@ -1048,6 +1111,22 @@ ProcessCopyOptions(ParseState *pstate, * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ errmsg("COPY %s requires %s to be set to %s", "REJECT_LIMIT", "ON_ERROR", "IGNORE"))); + + /* Check custom format routines */ + if (opts_out->format == COPY_FORMAT_CUSTOM) + { + if (is_from && opts_out->from_routine == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY format \"%s\" cannot be used with COPY FROM", + custom_format_name))); + + if (!is_from && opts_out->to_routine == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY format \"%s\" cannot be used with COPY TO", + custom_format_name))); + } } /* diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c new file mode 100644 index 00000000000..168efbcf30b --- /dev/null +++ b/src/backend/commands/copyapi.c @@ -0,0 +1,131 @@ +/*------------------------------------------------------------------------- + * + * copyapi.c + * Registry for pluggable COPY TO/FROM format handlers. + * + * The built-in formats (text, csv, binary, json) are dispatched directly by + * the COPY engine. Extensions can provide additional formats by registering + * a CopyToRoutine and/or CopyFromRoutine under a name from their _PG_init(); + * ProcessCopyOptions() then resolves "COPY ... (FORMAT 'name')" against this + * registry. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/commands/copyapi.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/copyapi.h" +#include "utils/memutils.h" + +typedef struct CopyCustomFormatEntry +{ + const char *name; /* constant string; never freed (see below) */ + const CopyToRoutine *to_routine; + const CopyFromRoutine *from_routine; + ProcessOneOptionFn option_fn; +} CopyCustomFormatEntry; + +static CopyCustomFormatEntry *CopyCustomFormatArray = NULL; +static int CopyCustomFormatsAssigned = 0; +static int CopyCustomFormatsAllocated = 0; + +/* Is 'name' one of the built-in format keywords? */ +static bool +is_builtin_copy_format(const char *name) +{ + return (strcmp(name, "text") == 0 || + strcmp(name, "csv") == 0 || + strcmp(name, "binary") == 0 || + strcmp(name, "json") == 0); +} + +/* + * Register a custom COPY format. Intended to be called from an extension's + * _PG_init(). Either routine may be NULL if the format does not support that + * direction (but not both). + * + * 'option_fn' may also be NULL if the format takes no format-specific options. + * + * 'name' is assumed to be a constant string or allocated in storage that will + * never be freed; it is stored by reference. + */ +void +RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, + const CopyFromRoutine *from, ProcessOneOptionFn option_fn) +{ + Assert(name != NULL && name[0] != '\0'); + + /* Must support at least one direction */ + Assert(to != NULL || from != NULL); + + Assert(to == NULL || + (to->CopyToStart != NULL && to->CopyToOneRow != NULL && + to->CopyToEnd != NULL)); + Assert(from == NULL || + (from->CopyFromStart != NULL && from->CopyFromOneRow != NULL && + from->CopyFromEnd != NULL)); + + /* Check if it's already used by built-in format names */ + if (is_builtin_copy_format(name)) + elog(ERROR, "COPY format \"%s\" is a built-in format name", name); + + /* Reject a duplicate registration. */ + for (int i = 0; i < CopyCustomFormatsAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].name, name) == 0) + elog(ERROR, "COPY format \"%s\" is already registered", name); + } + + /* Create the array on first use; it must outlive the current context. */ + if (CopyCustomFormatArray == NULL) + { + CopyCustomFormatsAllocated = 16; + CopyCustomFormatArray = (CopyCustomFormatEntry *) + MemoryContextAlloc(TopMemoryContext, + CopyCustomFormatsAllocated * sizeof(CopyCustomFormatEntry)); + } + + /* Expand if full. */ + if (CopyCustomFormatsAssigned >= CopyCustomFormatsAllocated) + { + CopyCustomFormatsAllocated *= 2; + CopyCustomFormatArray = (CopyCustomFormatEntry *) + repalloc_array(CopyCustomFormatArray, CopyCustomFormatEntry, CopyCustomFormatsAllocated); + } + + CopyCustomFormatArray[CopyCustomFormatsAssigned].name = name; + CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to; + CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from; + CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn; + CopyCustomFormatsAssigned++; +} + +/* + * Look up a previously registered custom format. Returns false if 'name' is + * not registered. Out-parameters may be NULL if not wanted. + */ +bool +GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn) +{ + for (int i = 0; i < CopyCustomFormatsAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].name, name) == 0) + { + if (to) + *to = CopyCustomFormatArray[i].to_routine; + if (from) + *from = CopyCustomFormatArray[i].from_routine; + if (option_fn) + *option_fn = CopyCustomFormatArray[i].option_fn; + + return true; + } + } + return false; +} diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 2c57b32f4de..69ec94c9ec1 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -158,7 +158,9 @@ static const CopyFromRoutine CopyFromRoutineBinary = { static const CopyFromRoutine * CopyFromGetRoutine(const CopyFormatOptions *opts) { - if (opts->format == COPY_FORMAT_CSV) + if (opts->format == COPY_FORMAT_CUSTOM) + return opts->from_routine; + else if (opts->format == COPY_FORMAT_CSV) return &CopyFromRoutineCSV; else if (opts->format == COPY_FORMAT_BINARY) return &CopyFromRoutineBinary; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index ef2038c9a5d..f897f23737f 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -130,7 +130,9 @@ static const CopyToRoutine CopyToRoutineBinary = { static const CopyToRoutine * CopyToGetRoutine(const CopyFormatOptions *opts) { - if (opts->format == COPY_FORMAT_CSV) + if (opts->format == COPY_FORMAT_CUSTOM) + return opts->to_routine; + else if (opts->format == COPY_FORMAT_CSV) return &CopyToRoutineCSV; else if (opts->format == COPY_FORMAT_BINARY) return &CopyToRoutineBinary; diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build index 9f258d566eb..d98273da67e 100644 --- a/src/backend/commands/meson.build +++ b/src/backend/commands/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'constraint.c', 'conversioncmds.c', 'copy.c', + 'copyapi.c', 'copyfrom.c', 'copyfromparse.c', 'copyto.c', diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 5e710efff5b..9c40ca4ba09 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -58,7 +58,16 @@ typedef enum CopyFormat COPY_FORMAT_BINARY, COPY_FORMAT_CSV, COPY_FORMAT_JSON, + COPY_FORMAT_CUSTOM, /* format provided by an extension */ } CopyFormat; +#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM) + +/* + * Full definitions live in commands/copyapi.h, which includes this header; + * CopyFormatOptions only needs to hold pointers to the resolved routines. + */ +struct CopyToRoutine; +struct CopyFromRoutine; /* * A struct to hold COPY options, in a parsed form. All of these are related @@ -97,6 +106,16 @@ typedef struct CopyFormatOptions CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ int64 reject_limit; /* maximum tolerable number of errors */ List *convert_select; /* list of column names (can be NIL) */ + + /* + * Resolved handler for a custom format. The directoin not in use may be + * NULL. For built-in formats these are unused. + */ + const struct CopyToRoutine *to_routine; + const struct CopyFromRoutine *from_routine; + + /* Custom format private option data */ + void *format_private_opts; } CopyFormatOptions; /* These are defined in copy_state.h */ diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h index 52cbf5067eb..6c5defbf4ee 100644 --- a/src/include/commands/copy_state.h +++ b/src/include/commands/copy_state.h @@ -178,6 +178,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* Custom format private data to store the state */ + void *format_private; } CopyFromStateData; /* @@ -248,6 +251,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* Custom format private data to store the state */ + void *format_private; } CopyToStateData; #endif /* COPY_STATE_H */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 398e7a78bb3..8eb5fe9c7dc 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,6 +14,7 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "commands/copy_state.h" #include "commands/copy.h" /* @@ -102,4 +103,40 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +/* + * Optional callback to process one format-specific COPY option. Invoked + * from ProcessCopyOptions() once per option that core did not recognize, after + * every core option has been parsed (so 'opts' is fully populated). + * + * Returns true if the option belongs to the format and is valid. Returns false + * if the option is not one the format recognizes, in which case core raises the + * "not accepted" error; thus an unrecognized option always errors, whether or + * not the format supplies this callback. For a recognized option with an invalid + * value, the callback should ereport() itself. + * + * 'pstate' may be NULL (e.g. when options are checked outside a real COPY, as + * file_fdw does); parser_errposition(pstate, ...) tolerates NULL. + */ +typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from, + DefElem *option); + +/* + * Register a COPY format under 'name', mapping it to its TO and/or FROM + * routines and optional option/validation callbacks. Intended to be called + * from an extension's _PG_init(). Either routine may be NULL if the format + * does not support that direction (but not both). Errors if 'name' collides + * with a built-in format or one already registered. + */ +extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, + const CopyFromRoutine *from, + ProcessOneOptionFn option_fn); + +/* + * Look up a previously registered custom format. Returns false if 'name' is + * not registered. Out-parameters may be NULL if not wanted. + */ +extern bool GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, + const CopyFromRoutine **from, + ProcessOneOptionFn * option_fn); + #endif /* COPYAPI_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1969d467c1d..5263710e451 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -540,6 +540,7 @@ ConvProcInfo ConversionLocation ConvertRowtypeExpr CookedConstraint +CopyCustomFormatEntry CopyDest CopyFormat CopyFormatOptions -- 2.54.0 [text/x-patch] v2-0004-Add-test-module-for-COPY-custom-format.patch (17.3K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/3-v2-0004-Add-test-module-for-COPY-custom-format.patch) download | inline diff: From d31d8cef68d10cb1817446af9a1e492ce88808e9 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 13:14:31 -0700 Subject: [PATCH v2 4/4] Add test module for COPY custom format. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + .../modules/test_copy_custom_format/Makefile | 20 +++ .../expected/test_copy_custom_format.out | 105 +++++++++++ .../test_copy_custom_format/meson.build | 32 ++++ .../sql/test_copy_custom_format.sql | 32 ++++ .../test_copy_custom_format.c | 169 ++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 361 insertions(+) create mode 100644 src/test/modules/test_copy_custom_format/Makefile create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out create mode 100644 src/test/modules/test_copy_custom_format/meson.build create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 0a74ab5c86f..6dcb66174f5 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -23,6 +23,7 @@ SUBDIRS = \ test_cloexec \ test_checksums \ test_copy_callbacks \ + test_copy_custom_format \ test_custom_rmgrs \ test_custom_stats \ test_custom_types \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 4bca42bb370..adfa413fe58 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -23,6 +23,7 @@ subdir('test_bloomfilter') subdir('test_cloexec') subdir('test_checksums') subdir('test_copy_callbacks') +subdir('test_copy_custom_format') subdir('test_cplusplusext') subdir('test_custom_rmgrs') subdir('test_custom_stats') diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile new file mode 100644 index 00000000000..68a2a04ff09 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/Makefile @@ -0,0 +1,20 @@ +# src/test/modules/test_copy_custom_format/Makefile + +MODULE_big = test_copy_custom_format +OBJS = \ + $(WIN32RES) \ + test_copy_custom_format.o +PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT" + +REGRESS = test_copy_custom_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_copy_custom_format +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out new file mode 100644 index 00000000000..817ca3fa60f --- /dev/null +++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out @@ -0,0 +1,105 @@ +LOAD 'test_copy_custom_format'; +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789); +COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 2 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToStart: the number of attributes of table: 1, the number of attributes to output: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized +ERROR: COPY format "nonexistent" not recognized +LINE 1: COPY copy_data TO stdout WITH (format 'nonexistent'); + ^ +COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting +ERROR: conflicting or redundant options +LINE 1: COPY copy_data TO stdout WITH (format 'text', format 'csv'); + ^ +COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR +ERROR: COPY format "test_format" does not accept option "bogus" +LINE 1: ...PY copy_data TO stdout WITH (format 'test_format', bogus 1); + ^ +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +ERROR: relation has 3 columns, exceeds max_attributes 2 +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive +ERROR: "max_attributes" must be a positive integer +COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR +ERROR: "max_attributes" must be a positive integer +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required +ERROR: max_attributes requires an integer value +COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate) +ERROR: FREEZE cannot be used with "disallow_freeze" option +COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +-- The built-in options are handled in the same way of built-in formats. +COPY copy_data TO stdout WITH (format 'test_format', delimiter ','); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', quote '"'); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO +ERROR: COPY FREEZE cannot be used with COPY TO diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build new file mode 100644 index 00000000000..a231ed57649 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/meson.build @@ -0,0 +1,32 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +test_copy_custom_format_sources = files( +'test_copy_custom_format.c', +) + +if host_system == 'windows' + test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_copy_custom_format', + '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',]) +endif + +test_copy_custom_format = shared_module('test_copy_custom_format', + test_copy_custom_format_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_copy_custom_format + +tests += { + 'name': 'test_copy_custom_format', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_copy_custom_format', + ], + # Disabled because these tests require + # "shared_preload_libraries=test_custom_copy_format", which typical + # runningcheck users do not have (e.g. buildfarm clients). + 'runningcheck': false, + }, +} diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql new file mode 100644 index 00000000000..59f58fa55a2 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql @@ -0,0 +1,32 @@ +LOAD 'test_copy_custom_format'; + +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789); + +COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End +COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End +\. + +COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2 +COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1 + +COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized +COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting + +COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR + +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2 +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive +COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required + +COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate) +COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK +\. + +-- The built-in options are handled in the same way of built-in formats. +COPY copy_data TO stdout WITH (format 'test_format', delimiter ','); +COPY copy_data TO stdout WITH (format 'test_format', quote '"'); +COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c new file mode 100644 index 00000000000..ca25832fcb0 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c @@ -0,0 +1,169 @@ +/*-------------------------------------------------------------------------- + * + * test_copy_custom_format.c + * Code for testing custom COPY format. + * + * Portions Copyright (c) 2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_copy_custom_format/test_copy_custom_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/copyapi.h" +#include "commands/copy_state.h" +#include "commands/defrem.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +typedef struct TestCopyOptions +{ + int max_attributes; + bool disallow_freeze; +} TestCopyOptions; + +static bool +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (t == NULL) + { + t = palloc0_object(TestCopyOptions); + opts->format_private_opts = (void *) t; + } + + if (strcmp(option->defname, "max_attributes") == 0) + { + int val = defGetInt32(option); + + if (val < 1) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"max_attributes\" must be a positive integer")); + + t->max_attributes = val; + return true; + } + else if (strcmp(option->defname, "disallow_freeze") == 0) + { + t->disallow_freeze = defGetBoolean(option); + return true; + } + + return false; +} + +static void +TestCopyValidateOptions(CopyFormatOptions *opts, bool is_from) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (!t) + return; + + if (t->disallow_freeze && opts->freeze) + ereport(ERROR, + errmsg("FREEZE cannot be used with \"disallow_freeze\" option")); +} + +static void +TestCopyFromInFunc(CopyFromState cstate, Oid atttypid, FmgrInfo *finfo, Oid *typioparam) +{ + ereport(NOTICE, + errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +check_max_attributes(CopyFormatOptions *opts, TupleDesc tupDesc) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (t != NULL && t->max_attributes > 0 && tupDesc->natts > t->max_attributes) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relation has %d columns, exceeds max_attributes %d", + tupDesc->natts, t->max_attributes)); +} + +static void +TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) +{ + check_max_attributes(&cstate->opts, tupDesc); + + ereport(NOTICE, + errmsg("CopyFromStart: the number of attributes of table: %d, the number of attributes to input: %d", + tupDesc->natts, list_length(cstate->attnumlist))); +} + +static bool +TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + ereport(NOTICE, errmsg("CopyFromOneRow")); + + return false; +} + +static void +TestCopyFromEnd(CopyFromState cstate) +{ + ereport(NOTICE, errmsg("CopyFromEnd")); +} + +static void +TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo) +{ + ereport(NOTICE, errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +TestCopyToStart(CopyToState cstate, TupleDesc tupDesc) +{ + check_max_attributes(&cstate->opts, tupDesc); + + ereport(NOTICE, + errmsg("CopyToStart: the number of attributes of table: %d, the number of attributes to output: %d", + tupDesc->natts, list_length(cstate->attnumlist))); +} + +static void +TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid))); +} + +static void +TestCopyToEnd(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToEnd"))); +} + +static const CopyToRoutine TestCopyToRoutine = { + .CopyToOutFunc = TestCopyToOutFunc, + .CopyToStart = TestCopyToStart, + .CopyToOneRow = TestCopyToOneRow, + .CopyToEnd = TestCopyToEnd, +}; + + +static const CopyFromRoutine TestCopyFromRoutine = { + .CopyFromInFunc = TestCopyFromInFunc, + .CopyFromStart = TestCopyFromStart, + .CopyFromOneRow = TestCopyFromOneRow, + .CopyFromEnd = TestCopyFromEnd, +}; + +void +_PG_init(void) +{ + RegisterCopyCustomFormat("test_format", + &TestCopyToRoutine, + &TestCopyFromRoutine, + &TestCopyProcessOneOption, + &TestCopyValidateOptions); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5263710e451..552669abc5f 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3179,6 +3179,7 @@ Tcl_Obj Tcl_Size Tcl_Time TempNamespaceStatus +TestCopyOptions TestDSMRegistryHashEntry TestDSMRegistryStruct TestDecodingData -- 2.54.0 [text/x-patch] v2-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch (5.6K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/4-v2-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch) download | inline diff: From c0e1e5986d10125b41f3fde10cdb3ba931db00b3 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 09:23:09 -0700 Subject: [PATCH v2 3/4] Add an hook for custom COPY format option validation. Author: Reviewed-by: Discussion: https://postgr.es/m/ --- src/backend/commands/copy.c | 13 ++++++++++++- src/backend/commands/copyapi.c | 10 ++++++++-- src/include/commands/copyapi.h | 15 +++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 2fdba026ee0..45908d0c1e5 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -599,6 +599,7 @@ ProcessCopyOptions(ParseState *pstate, */ List *deferred_options = NIL; ProcessOneOptionFn custom_process_option_fn = NULL; + ValidateOptionsFn custom_validate_options_fn = NULL; char *custom_format_name = NULL; /* Support external use for option sanity checking */ @@ -631,7 +632,8 @@ ProcessCopyOptions(ParseState *pstate, opts_out->format = COPY_FORMAT_JSON; else if (GetCopyCustomFormatRoutines(fmt, &opts_out->to_routine, &opts_out->from_routine, - &custom_process_option_fn)) + &custom_process_option_fn, + &custom_validate_options_fn)) { opts_out->format = COPY_FORMAT_CUSTOM; custom_format_name = fmt; @@ -1126,6 +1128,15 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("COPY format \"%s\" cannot be used with COPY TO", custom_format_name))); + + /* + * Let the format validate its fully-parsed options as a whole. This + * runs even when no format-specific options were given, so a format + * can reject incompatible core options or enforce cross-option + * constraints. + */ + if (custom_validate_options_fn != NULL) + custom_validate_options_fn(opts_out, is_from); } } diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c index 168efbcf30b..0bd1cb71af2 100644 --- a/src/backend/commands/copyapi.c +++ b/src/backend/commands/copyapi.c @@ -28,6 +28,7 @@ typedef struct CopyCustomFormatEntry const CopyToRoutine *to_routine; const CopyFromRoutine *from_routine; ProcessOneOptionFn option_fn; + ValidateOptionsFn validate_fn; } CopyCustomFormatEntry; static CopyCustomFormatEntry *CopyCustomFormatArray = NULL; @@ -56,7 +57,8 @@ is_builtin_copy_format(const char *name) */ void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, - const CopyFromRoutine *from, ProcessOneOptionFn option_fn) + const CopyFromRoutine *from, ProcessOneOptionFn option_fn, + ValidateOptionsFn validate_fn) { Assert(name != NULL && name[0] != '\0'); @@ -102,6 +104,7 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to; CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from; CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn; + CopyCustomFormatArray[CopyCustomFormatsAssigned].validate_fn = validate_fn; CopyCustomFormatsAssigned++; } @@ -111,7 +114,8 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, */ bool GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, - const CopyFromRoutine **from, ProcessOneOptionFn * option_fn) + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn, + ValidateOptionsFn * validate_fn) { for (int i = 0; i < CopyCustomFormatsAssigned; i++) { @@ -123,6 +127,8 @@ GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, *from = CopyCustomFormatArray[i].from_routine; if (option_fn) *option_fn = CopyCustomFormatArray[i].option_fn; + if (validate_fn) + *validate_fn = CopyCustomFormatArray[i].validate_fn; return true; } diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 8eb5fe9c7dc..c47c89a858f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -120,6 +120,15 @@ typedef struct CopyFromRoutine typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from, DefElem *option); +/* + * Optional callback to validate a custom format's fully-parsed options as a + * whole. Invoked once from ProcessCopyOptions() after all options have been + * processed, so it can enforce cross-option constraints and reject + * incompatible core options. It runs even when no format-specific options were + * supplied. Reports problems with ereport(). + */ +typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from); + /* * Register a COPY format under 'name', mapping it to its TO and/or FROM * routines and optional option/validation callbacks. Intended to be called @@ -129,7 +138,8 @@ typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from, */ extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, const CopyFromRoutine *from, - ProcessOneOptionFn option_fn); + ProcessOneOptionFn option_fn, + ValidateOptionsFn validate_fn); /* * Look up a previously registered custom format. Returns false if 'name' is @@ -137,6 +147,7 @@ extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, */ extern bool GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, const CopyFromRoutine **from, - ProcessOneOptionFn * option_fn); + ProcessOneOptionFn * option_fn, + ValidateOptionsFn * validate_fn); #endif /* COPYAPI_H */ -- 2.54.0 [text/x-patch] v2-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch (26.4K, ../../CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com/5-v2-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch) download | inline diff: From 47aa927d65e483331f9b2010499b4c7f0bfb562c Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 11:50:27 -0700 Subject: [PATCH v2 1/4] Move Copy[From|To]StateData to copy_state.h. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- contrib/file_fdw/file_fdw.c | 2 +- src/backend/commands/copyfrom.c | 5 +- src/backend/commands/copyfromparse.c | 10 +- src/backend/commands/copyto.c | 88 +------- src/include/commands/copy.h | 2 +- src/include/commands/copy_state.h | 253 +++++++++++++++++++++++ src/include/commands/copyfrom_internal.h | 165 +-------------- 7 files changed, 273 insertions(+), 252 deletions(-) create mode 100644 src/include/commands/copy_state.h diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 33a37d832ce..d152d05b92e 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -22,7 +22,7 @@ #include "catalog/pg_authid.h" #include "catalog/pg_foreign_table.h" #include "commands/copy.h" -#include "commands/copyfrom_internal.h" +#include "commands/copy_state.h" #include "commands/defrem.h" #include "commands/explain_format.h" #include "commands/explain_state.h" diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 0087585b2c4..2c57b32f4de 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -30,6 +30,7 @@ #include "access/xact.h" #include "catalog/namespace.h" #include "commands/copyapi.h" +#include "commands/copy_state.h" #include "commands/copyfrom_internal.h" #include "commands/progress.h" #include "commands/trigger.h" @@ -1732,7 +1733,7 @@ BeginCopyFrom(ParseState *pstate, pg_encoding_to_char(GetDatabaseEncoding())))); } - cstate->copy_src = COPY_FILE; /* default */ + cstate->copy_src = COPY_SOURCE_FILE; /* default */ cstate->whereClause = whereClause; @@ -1861,7 +1862,7 @@ BeginCopyFrom(ParseState *pstate, if (data_source_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_src = COPY_CALLBACK; + cstate->copy_src = COPY_SOURCE_CALLBACK; cstate->data_source_cb = data_source_cb; } else if (pipe) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 65fd5a0ab4f..0ff5db4b62d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -184,7 +184,7 @@ ReceiveCopyBegin(CopyFromState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_src = COPY_FRONTEND; + cstate->copy_src = COPY_SOURCE_FRONTEND; cstate->fe_msgbuf = makeStringInfo(); /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); @@ -252,7 +252,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) switch (cstate->copy_src) { - case COPY_FILE: + case COPY_SOURCE_FILE: pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ); bytesread = fread(databuf, 1, maxread, cstate->copy_file); pgstat_report_wait_end(); @@ -263,7 +263,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) if (bytesread == 0) cstate->raw_reached_eof = true; break; - case COPY_FRONTEND: + case COPY_SOURCE_FRONTEND: while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) { int avail; @@ -346,7 +346,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) bytesread += avail; } break; - case COPY_CALLBACK: + case COPY_SOURCE_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); break; } @@ -1259,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (cstate->copy_src == COPY_SOURCE_FRONTEND) { int inbytes; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6755bb698de..ef2038c9a5d 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -23,8 +23,8 @@ #include "access/tupconvert.h" #include "catalog/pg_inherits.h" #include "commands/copyapi.h" +#include "commands/copy_state.h" #include "commands/progress.h" -#include "executor/execdesc.h" #include "executor/executor.h" #include "executor/tuptable.h" #include "funcapi.h" @@ -42,76 +42,6 @@ #include "utils/snapmgr.h" #include "utils/wait_event.h" -/* - * Represents the different dest cases we need to worry about at - * the bottom level - */ -typedef enum CopyDest -{ - COPY_FILE, /* to file (or a piped program) */ - COPY_FRONTEND, /* to frontend */ - COPY_CALLBACK, /* to callback function */ -} CopyDest; - -/* - * This struct contains all the state variables used throughout a COPY TO - * operation. - * - * Multi-byte encodings: all supported client-side encodings encode multi-byte - * characters by having the first byte's high bit set. Subsequent bytes of the - * character can have the high bit not set. When scanning data in such an - * encoding to look for a match to a single-byte (ie ASCII) character, we must - * use the full pg_encoding_mblen() machinery to skip over multibyte - * characters, else we might find a false match to a trailing byte. In - * supported server encodings, there is no possibility of a false match, and - * it's faster to make useless comparisons to trailing bytes than it is to - * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true - * when we have to do it the hard way. - */ -typedef struct CopyToStateData -{ - /* format-specific routines */ - const CopyToRoutine *routine; - - /* low-level state data */ - CopyDest copy_dest; /* type of copy source/destination */ - FILE *copy_file; /* used if copy_dest == COPY_FILE */ - StringInfo fe_msgbuf; /* used for all dests during COPY TO */ - - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - bool json_row_delim_needed; /* need delimiter before next row */ - StringInfo json_buf; /* reusable buffer for JSON output, - * initialized in BeginCopyTo */ - TupleDesc tupDesc; /* Descriptor for JSON output; for a column - * list this is a projected descriptor */ - Datum *json_projvalues; /* pre-allocated projection values, or - * NULL */ - bool *json_projnulls; /* pre-allocated projection nulls, or NULL */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - List *partitions; /* OID list of partitions to copy data from */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - FmgrInfo *out_functions; /* lookup info for output functions */ - MemoryContext rowcontext; /* per-row evaluation context */ - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; - /* DestReceiver for COPY (query) TO */ typedef struct { @@ -559,7 +489,7 @@ SendCopyBegin(CopyToState cstate) } pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->copy_dest = COPY_DEST_FRONTEND; } static void @@ -606,7 +536,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: pgstat_report_wait_start(WAIT_EVENT_COPY_TO_WRITE); if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || @@ -642,11 +572,11 @@ CopySendEndOfRow(CopyToState cstate) } pgstat_report_wait_end(); break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; - case COPY_CALLBACK: + case COPY_DEST_CALLBACK: cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len); break; } @@ -667,7 +597,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) { switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: /* Default line termination depends on platform */ #ifndef WIN32 CopySendChar(cstate, '\n'); @@ -675,7 +605,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) CopySendString(cstate, "\r\n"); #endif break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); break; @@ -1135,12 +1065,12 @@ BeginCopyTo(ParseState *pstate, /* See Multibyte encoding comment above */ cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - cstate->copy_dest = COPY_FILE; /* default */ + cstate->copy_dest = COPY_DEST_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->copy_dest = COPY_DEST_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index abecfe51098..5e710efff5b 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -99,7 +99,7 @@ typedef struct CopyFormatOptions List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; -/* These are private in commands/copy[from|to].c */ +/* These are defined in copy_state.h */ typedef struct CopyFromStateData *CopyFromState; typedef struct CopyToStateData *CopyToState; diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h new file mode 100644 index 00000000000..52cbf5067eb --- /dev/null +++ b/src/include/commands/copy_state.h @@ -0,0 +1,253 @@ +/*------------------------------------------------------------------------- + * + * copy_state.h + * prototypes for COPY TO/COPY FROM execution state. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994-5, Regents of the University of California + * + * src/include/commands/copy_state.h + * + *------------------------------------------------------------------------- + */ + +#ifndef COPY_STATE_H +#define COPY_STATE_H + +#include "commands/copy.h" +#include "commands/trigger.h" +#include "executor/execdesc.h" +#include "nodes/miscnodes.h" + +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_SOURCE_FILE, /* from file (or a piped program) */ + COPY_SOURCE_FRONTEND, /* from frontend */ + COPY_SOURCE_CALLBACK, /* from callback function */ +} CopySource; + +/* + * Represents the end-of-line terminator type of the input + */ +typedef enum EolType +{ + EOL_UNKNOWN, + EOL_NL, + EOL_CR, + EOL_CRNL, +} EolType; + +/* + * This struct contains all the state variables used throughout a COPY FROM + * operation. + */ +typedef struct CopyFromStateData +{ + /* format routine */ + const struct CopyFromRoutine *routine; + + /* low-level state data */ + CopySource copy_src; /* type of copy source */ + FILE *copy_file; /* used if copy_src == COPY_SOURCE_FILE */ + StringInfo fe_msgbuf; /* used if copy_src == COPY_SOURCE_FRONTEND */ + + EolType eol_type; /* EOL type of input */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + Oid conversion_proc; /* encoding conversion function */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + CopyFormatOptions opts; + bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ + Node *whereClause; /* WHERE condition (or NULL) */ + + /* these are just for error messages, see CopyFromErrorCallback */ + const char *cur_relname; /* table name for error messages */ + uint64 cur_lineno; /* line number for error messages */ + const char *cur_attname; /* current att for error messages */ + const char *cur_attval; /* current att value for error messages */ + bool relname_only; /* don't output line number, att, etc. */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + AttrNumber num_defaults; /* count of att that are missing and have + * default value */ + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapped during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ + int *defmap; /* array of default att numbers related to + * missing att */ + ExprState **defexprs; /* array of default att expressions for all + * att */ + bool *defaults; /* if DEFAULT marker was found for + * corresponding att */ + bool simd_enabled; /* use SIMD to scan for special chars? */ + + /* + * True if the corresponding attribute's is a constrained domain. This + * will be populated only when ON_ERROR is SET_NULL, otherwise NULL. + */ + bool *domain_with_constraint; + + bool volatile_defexprs; /* is any of defexprs volatile? */ + List *range_table; /* single element list of RangeTblEntry */ + List *rteperminfos; /* single element list of RTEPermissionInfo */ + ExprState *qualexpr; + + TransitionCaptureState *transition_capture; + + /* + * These variables are used to reduce overhead in COPY FROM. + * + * attribute_buf holds the separated, de-escaped text for each field of + * the current line. The CopyReadAttributes functions return arrays of + * pointers into this buffer. We avoid palloc/pfree overhead by re-using + * the buffer on each cycle. + * + * In binary COPY FROM, attribute_buf holds the binary data for the + * current field, but the usage is otherwise similar. + */ + StringInfoData attribute_buf; + + /* field raw data pointers found by COPY FROM */ + + int max_fields; + char **raw_fields; + + /* + * Similarly, line_buf holds the whole input line being processed. The + * input cycle is first to read the whole line into line_buf, and then + * extract the individual attribute fields into attribute_buf. line_buf + * is preserved unmodified so that we can display it in error messages if + * appropriate. (In binary mode, line_buf is not used.) + */ + StringInfoData line_buf; + bool line_buf_valid; /* contains the row being processed? */ + + /* + * input_buf holds input data, already converted to database encoding. + * + * In text mode, CopyReadLine parses this data sufficiently to locate line + * boundaries, then transfers the data to line_buf. We guarantee that + * there is a \0 at input_buf[input_buf_len] at all times. (In binary + * mode, input_buf is not used.) + * + * If encoding conversion is not required, input_buf is not a separate + * buffer but points directly to raw_buf. In that case, input_buf_len + * tracks the number of bytes that have been verified as valid in the + * database encoding, and raw_buf_len is the total number of bytes stored + * in the buffer. + */ +#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ + char *input_buf; + int input_buf_index; /* next byte to process */ + int input_buf_len; /* total # of bytes stored */ + bool input_reached_eof; /* true if we reached EOF */ + bool input_reached_error; /* true if a conversion error happened */ + /* Shorthand for number of unconsumed bytes available in input_buf */ +#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) + + /* + * raw_buf holds raw input data read from the data source (file or client + * connection), not yet converted to the database encoding. Like with + * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. + */ +#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ + char *raw_buf; + int raw_buf_index; /* next byte to process */ + int raw_buf_len; /* total # of bytes stored */ + bool raw_reached_eof; /* true if we reached EOF */ + + /* Shorthand for number of unconsumed bytes available in raw_buf */ +#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) + + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyFromStateData; + +/* + * Represents the different dest cases we need to worry about at + * the bottom level + */ +typedef enum CopyDest +{ + COPY_DEST_FILE, /* to file (or a piped program) */ + COPY_DEST_FRONTEND, /* to frontend */ + COPY_DEST_CALLBACK, /* to callback function */ +} CopyDest; + +/* + * This struct contains all the state variables used throughout a COPY TO + * operation. + * + * Multi-byte encodings: all supported client-side encodings encode multi-byte + * characters by having the first byte's high bit set. Subsequent bytes of the + * character can have the high bit not set. When scanning data in such an + * encoding to look for a match to a single-byte (ie ASCII) character, we must + * use the full pg_encoding_mblen() machinery to skip over multibyte + * characters, else we might find a false match to a trailing byte. In + * supported server encodings, there is no possibility of a false match, and + * it's faster to make useless comparisons to trailing bytes than it is to + * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true + * when we have to do it the hard way. + */ +typedef struct CopyToStateData +{ + /* format-specific routines */ + const struct CopyToRoutine *routine; + + /* low-level state data */ + CopyDest copy_dest; /* type of copy source/destination */ + FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + bool json_row_delim_needed; /* need delimiter before next row */ + StringInfo json_buf; /* reusable buffer for JSON output, + * initialized in BeginCopyTo */ + TupleDesc tupDesc; /* Descriptor for JSON output; for a column + * list this is a projected descriptor */ + Datum *json_projvalues; /* pre-allocated projection values, or + * NULL */ + bool *json_projnulls; /* pre-allocated projection nulls, or NULL */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + List *partitions; /* OID list of partitions to copy data from */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *out_functions; /* lookup info for output functions */ + MemoryContext rowcontext; /* per-row evaluation context */ + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyToStateData; + +#endif /* COPY_STATE_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 9d3e244ee55..f7afade9a39 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -14,31 +14,7 @@ #ifndef COPYFROM_INTERNAL_H #define COPYFROM_INTERNAL_H -#include "commands/copy.h" -#include "commands/trigger.h" -#include "nodes/miscnodes.h" - -/* - * Represents the different source cases we need to worry about at - * the bottom level - */ -typedef enum CopySource -{ - COPY_FILE, /* from file (or a piped program) */ - COPY_FRONTEND, /* from frontend */ - COPY_CALLBACK, /* from callback function */ -} CopySource; - -/* - * Represents the end-of-line terminator type of the input - */ -typedef enum EolType -{ - EOL_UNKNOWN, - EOL_NL, - EOL_CR, - EOL_CRNL, -} EolType; +#include "commands/copy_state.h" /* * Represents the insert method to be used during COPY FROM. @@ -52,145 +28,6 @@ typedef enum CopyInsertMethod * ExecForeignBatchInsert only if valid */ } CopyInsertMethod; -/* - * This struct contains all the state variables used throughout a COPY FROM - * operation. - */ -typedef struct CopyFromStateData -{ - /* format routine */ - const struct CopyFromRoutine *routine; - - /* low-level state data */ - CopySource copy_src; /* type of copy source */ - FILE *copy_file; /* used if copy_src == COPY_FILE */ - StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ - - EolType eol_type; /* EOL type of input */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - Oid conversion_proc; /* encoding conversion function */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; - bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ - Node *whereClause; /* WHERE condition (or NULL) */ - - /* these are just for error messages, see CopyFromErrorCallback */ - const char *cur_relname; /* table name for error messages */ - uint64 cur_lineno; /* line number for error messages */ - const char *cur_attname; /* current att for error messages */ - const char *cur_attval; /* current att value for error messages */ - bool relname_only; /* don't output line number, att, etc. */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - AttrNumber num_defaults; /* count of att that are missing and have - * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ - ErrorSaveContext *escontext; /* soft error trapped during in_functions - * execution */ - uint64 num_errors; /* total number of rows which contained soft - * errors */ - int *defmap; /* array of default att numbers related to - * missing att */ - ExprState **defexprs; /* array of default att expressions for all - * att */ - bool *defaults; /* if DEFAULT marker was found for - * corresponding att */ - bool simd_enabled; /* use SIMD to scan for special chars? */ - - /* - * True if the corresponding attribute's is a constrained domain. This - * will be populated only when ON_ERROR is SET_NULL, otherwise NULL. - */ - bool *domain_with_constraint; - - bool volatile_defexprs; /* is any of defexprs volatile? */ - List *range_table; /* single element list of RangeTblEntry */ - List *rteperminfos; /* single element list of RTEPermissionInfo */ - ExprState *qualexpr; - - TransitionCaptureState *transition_capture; - - /* - * These variables are used to reduce overhead in COPY FROM. - * - * attribute_buf holds the separated, de-escaped text for each field of - * the current line. The CopyReadAttributes functions return arrays of - * pointers into this buffer. We avoid palloc/pfree overhead by re-using - * the buffer on each cycle. - * - * In binary COPY FROM, attribute_buf holds the binary data for the - * current field, but the usage is otherwise similar. - */ - StringInfoData attribute_buf; - - /* field raw data pointers found by COPY FROM */ - - int max_fields; - char **raw_fields; - - /* - * Similarly, line_buf holds the whole input line being processed. The - * input cycle is first to read the whole line into line_buf, and then - * extract the individual attribute fields into attribute_buf. line_buf - * is preserved unmodified so that we can display it in error messages if - * appropriate. (In binary mode, line_buf is not used.) - */ - StringInfoData line_buf; - bool line_buf_valid; /* contains the row being processed? */ - - /* - * input_buf holds input data, already converted to database encoding. - * - * In text mode, CopyReadLine parses this data sufficiently to locate line - * boundaries, then transfers the data to line_buf. We guarantee that - * there is a \0 at input_buf[input_buf_len] at all times. (In binary - * mode, input_buf is not used.) - * - * If encoding conversion is not required, input_buf is not a separate - * buffer but points directly to raw_buf. In that case, input_buf_len - * tracks the number of bytes that have been verified as valid in the - * database encoding, and raw_buf_len is the total number of bytes stored - * in the buffer. - */ -#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ - char *input_buf; - int input_buf_index; /* next byte to process */ - int input_buf_len; /* total # of bytes stored */ - bool input_reached_eof; /* true if we reached EOF */ - bool input_reached_error; /* true if a conversion error happened */ - /* Shorthand for number of unconsumed bytes available in input_buf */ -#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) - - /* - * raw_buf holds raw input data read from the data source (file or client - * connection), not yet converted to the database encoding. Like with - * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. - */ -#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ - char *raw_buf; - int raw_buf_index; /* next byte to process */ - int raw_buf_len; /* total # of bytes stored */ - bool raw_reached_eof; /* true if we reached EOF */ - - /* Shorthand for number of unconsumed bytes available in raw_buf */ -#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) - - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; - extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); -- 2.54.0 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2026-06-23 05:41 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Sutou Kouhei @ 2026-06-23 05:41 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, Thanks for restarting this. In <CAD21AoCnA7vayZAOmwVqTSOyWfyBhyxH7mBb4UzjskF-eZ+_Jg@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 22 Jun 2026 18:06:07 -0700, Masahiko Sawada <[email protected]> wrote: > After more thought, I'd like to keep the custom-format changes to the > bare minimum and not disturb the existing built-in format processing. +1 > Updated patches attached: > > - 0001 moves CopyFromStateData and CopyToStateData to a new > copy_state.h, so extensions can implement their routines without > including the *_internal.h headers. It also drops file_fdw.c's > dependency on copyfrom_internal.h. +1 > - 0002 introduces the registration API and the opaque per-format > pointer in both structs. > --- /dev/null > +++ b/src/backend/commands/copyapi.c > +bool > +GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, > + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn) How about returning CopyCustomFormatEntry instead? The function name is "Get...Routines" but it also returns ProcessOneOptionFn. "Get...Routines" is a bit strange. > --- a/src/include/commands/copyapi.h > +++ b/src/include/commands/copyapi.h > @@ -102,4 +103,40 @@ typedef struct CopyFromRoutine > ... > +typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from, > + DefElem *option); How about adding "Copy" keyword to the type name such as "ProcessOneCopyOptionFn" because this is only for COPY format? > --- a/src/include/commands/copy.h > +++ b/src/include/commands/copy.h > @@ -58,7 +58,16 @@ typedef enum CopyFormat > ... > +#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM) How about renaming this to CopyFormatIsBuiltin() or something? "...Builtins" is a bit strange because this returns a boolean. > - 0003 adds a callback to validate the COPY options as a whole, called > after all options are processed. > --- a/src/include/commands/copyapi.h > +++ b/src/include/commands/copyapi.h > @@ -120,6 +120,15 @@ typedef struct CopyFromRoutine > ... > +typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from); How about adding "Copy" keyword like "ValidateCopyOptionsFn"? > - 0004 adds the regression tests. > --- /dev/null > +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c > @@ -0,0 +1,169 @@ > ... > +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option) > +{ > + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; > + > + if (t == NULL) > + { > + t = palloc0_object(TestCopyOptions); > + opts->format_private_opts = (void *) t; > + } This is not a blocker but we may want to add InitializeCopyOptions callback for this. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2026-06-24 01:15 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 127+ messages in thread From: Masahiko Sawada @ 2026-06-24 01:15 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jun 22, 2026 at 10:41 PM Sutou Kouhei <[email protected]> wrote: > > > - 0002 introduces the registration API and the opaque per-format > > pointer in both structs. > > > --- /dev/null > > +++ b/src/backend/commands/copyapi.c > > > +bool > > +GetCopyCustomFormatRoutines(const char *name, const CopyToRoutine **to, > > + const CopyFromRoutine **from, ProcessOneOptionFn * option_fn) > > How about returning CopyCustomFormatEntry instead? The > function name is "Get...Routines" but it also returns > ProcessOneOptionFn. "Get...Routines" is a bit strange. Agreed. > > > --- a/src/include/commands/copyapi.h > > +++ b/src/include/commands/copyapi.h > > > @@ -102,4 +103,40 @@ typedef struct CopyFromRoutine > > ... > > +typedef bool (*ProcessOneOptionFn) (CopyFormatOptions *opts, bool is_from, > > + DefElem *option); > > How about adding "Copy" keyword to the type name such as > "ProcessOneCopyOptionFn" because this is only for COPY format? Agreed. > > > --- a/src/include/commands/copy.h > > +++ b/src/include/commands/copy.h > > > @@ -58,7 +58,16 @@ typedef enum CopyFormat > > ... > > +#define CopyFormatBuiltins(format) ((format) != COPY_FORMAT_CUSTOM) > > How about renaming this to CopyFormatIsBuiltin() or > something? "...Builtins" is a bit strange because this > returns a boolean. Agreed. > > > - 0003 adds a callback to validate the COPY options as a whole, called > > after all options are processed. > > > --- a/src/include/commands/copyapi.h > > +++ b/src/include/commands/copyapi.h > > @@ -120,6 +120,15 @@ typedef struct CopyFromRoutine > > ... > > +typedef void (*ValidateOptionsFn) (CopyFormatOptions *opts, bool is_from); > > How about adding "Copy" keyword like "ValidateCopyOptionsFn"? Agreed. > > > - 0004 adds the regression tests. > > > --- /dev/null > > +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c > > > @@ -0,0 +1,169 @@ > > ... > > +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option) > > +{ > > + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; > > + > > + if (t == NULL) > > + { > > + t = palloc0_object(TestCopyOptions); > > + opts->format_private_opts = (void *) t; > > + } > > This is not a blocker but we may want to add > InitializeCopyOptions callback for this. It would save just a few lines. It might be worth having the initialization callback if it would enable extensions to do what cannot be done with the current proposed callbacks. Thank you for reviewing the patches! I've attached updated patches. I'll verify that the new API works well with an experimental custom copy format extension. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [text/x-patch] v3-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch (3.6K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/2-v3-0003-Add-an-hook-for-custom-COPY-format-option-validat.patch) download | inline diff: From 43c06a26a14c914f6bc3d72d01accd95e3b0f8e8 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 09:23:09 -0700 Subject: [PATCH v3 3/4] Add an hook for custom COPY format option validation. Author: Reviewed-by: Discussion: https://postgr.es/m/ --- src/backend/commands/copy.c | 9 +++++++++ src/backend/commands/copyapi.c | 4 +++- src/include/commands/copyapi.h | 14 +++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 0fda54905cd..dab523c68db 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1119,6 +1119,15 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("COPY format \"%s\" cannot be used with COPY TO", opts_out->custom_format_ent->name))); + + /* + * Let the format validate its fully-parsed options as a whole. This + * runs even when no format-specific options were given, so a format + * can reject incompatible core options or enforce cross-option + * constraints. + */ + if (opts_out->custom_format_ent->validate_fn != NULL) + opts_out->custom_format_ent->validate_fn(opts_out, is_from); } } diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c index f007d73e867..29fb98ff4c2 100644 --- a/src/backend/commands/copyapi.c +++ b/src/backend/commands/copyapi.c @@ -48,7 +48,8 @@ is_builtin_copy_format(const char *name) */ void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, - const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn) + const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn, + ValidateCopyOptionsFn validate_fn) { Assert(name != NULL && name[0] != '\0'); @@ -94,6 +95,7 @@ RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to; CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from; CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn; + CopyCustomFormatArray[CopyCustomFormatsAssigned].validate_fn = validate_fn; CopyCustomFormatsAssigned++; } diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1b323b23bba..f1924424df8 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -117,6 +117,15 @@ typedef struct CopyFromRoutine typedef bool (*ProcessOneCopyOptionFn) (CopyFormatOptions *opts, bool is_from, DefElem *option); +/* + * Optional callback to validate a custom format's fully-parsed options as a + * whole. Invoked once from ProcessCopyOptions() after all options have been + * processed, so it can enforce cross-option constraints and reject + * incompatible core options. It runs even when no format-specific options were + * supplied. Reports problems with ereport(). + */ +typedef void (*ValidateCopyOptionsFn) (CopyFormatOptions *opts, bool is_from); + /* * Sturct to store the registered custom format information. */ @@ -126,11 +135,14 @@ typedef struct CopyCustomFormatEntry const CopyToRoutine *to_routine; const CopyFromRoutine *from_routine; ProcessOneCopyOptionFn option_fn; + ValidateCopyOptionsFn validate_fn; } CopyCustomFormatEntry; extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, const CopyFromRoutine *from, - ProcessOneCopyOptionFn option_fn); + ProcessOneCopyOptionFn option_fn, + ValidateCopyOptionsFn validate_fn); + extern const CopyCustomFormatEntry *GetCopyCustomFormatRoutines(const char *name); #endif /* COPYAPI_H */ -- 2.54.0 [text/x-patch] v3-0002-Allow-extensions-to-register-custom-format-to-COP.patch (18.1K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/3-v3-0002-Allow-extensions-to-register-custom-format-to-COP.patch) download | inline diff: From 4b1bbfb28daaa8a177b3a802572ae930fc15d199 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 09:21:51 -0700 Subject: [PATCH v3 2/4] Allow extensions to register custom format to COPY TO and COPY FROM. Author: Reviewed-by: Discussion: https://postgr.es/m/ --- src/backend/commands/Makefile | 1 + src/backend/commands/copy.c | 90 ++++++++++++++++++++--- src/backend/commands/copyapi.c | 114 ++++++++++++++++++++++++++++++ src/backend/commands/copyfrom.c | 4 +- src/backend/commands/copyto.c | 4 +- src/backend/commands/meson.build | 1 + src/include/commands/copy.h | 18 +++++ src/include/commands/copy_state.h | 6 ++ src/include/commands/copyapi.h | 31 ++++++++ src/tools/pgindent/typedefs.list | 1 + 10 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 src/backend/commands/copyapi.c diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 5b9d084977e..17b7aa08b55 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -23,6 +23,7 @@ OBJS = \ constraint.o \ conversioncmds.o \ copy.o \ + copyapi.o \ copyfrom.o \ copyfromparse.o \ copyto.o \ diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 003b70852bb..0fda54905cd 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -23,6 +23,7 @@ #include "access/xact.h" #include "catalog/pg_authid.h" #include "commands/copy.h" +#include "commands/copyapi.h" #include "commands/defrem.h" #include "executor/executor.h" #include "mb/pg_wchar.h" @@ -592,6 +593,12 @@ ProcessCopyOptions(ParseState *pstate, bool force_array_specified = false; ListCell *option; + /* + * Options not recognized by core are collected here and, once the format + * is known, either handed to a custom format's option parser or rejected. + */ + List *deferred_options = NIL; + /* Support external use for option sanity checking */ if (opts_out == NULL) opts_out = palloc0_object(CopyFormatOptions); @@ -620,6 +627,8 @@ ProcessCopyOptions(ParseState *pstate, opts_out->format = COPY_FORMAT_BINARY; else if (strcmp(fmt, "json") == 0) opts_out->format = COPY_FORMAT_JSON; + else if ((opts_out->custom_format_ent = GetCopyCustomFormatRoutines(fmt)) != NULL) + opts_out->format = COPY_FORMAT_CUSTOM; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -775,11 +784,54 @@ ProcessCopyOptions(ParseState *pstate, opts_out->reject_limit = defGetCopyRejectLimitOption(defel); } else + { + /* + * Not a core option. Defer the check to after the loop as it may + * belong to a custom format whose "format" option has not been + * seen yet. + */ + deferred_options = lappend(deferred_options, defel); + } + } + + /* + * Now that the format and every option have been seen, resolve the + * deferred options. + */ + if (deferred_options != NIL) + { + /* + * For a custom format, they belong to the handler; for any built-in + * (including the default) an unrecognized option is an error, + * preserving the historical behavior relied on by external callers + * such as file_fdw. + */ + if (opts_out->format != COPY_FORMAT_CUSTOM || opts_out->custom_format_ent->option_fn == NULL) + { + DefElem *defel = linitial_node(DefElem, deferred_options); + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("option \"%s\" not recognized", defel->defname), parser_errposition(pstate, defel->location))); + } + + /* + * Hand each option core did not recognize to the format's per-option + * callback. Anything the format does not claim (or any option at all + * if it has no callback) is an error, so an unrecognized option + * always fails here. + */ + foreach_node(DefElem, opt, deferred_options) + { + if (!opts_out->custom_format_ent->option_fn(opts_out, is_from, opt)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("COPY format \"%s\" does not accept option \"%s\"", + opts_out->custom_format_ent->name, opt->defname), + parser_errposition(pstate, opt->location))); + } } /* @@ -869,7 +921,7 @@ ProcessCopyOptions(ParseState *pstate, * future-proofing. Likewise we disallow all digits though only octal * digits are actually dangerous. */ - if (opts_out->format != COPY_FORMAT_CSV && + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", opts_out->delim[0]) != NULL) ereport(ERROR, @@ -888,7 +940,8 @@ ProcessCopyOptions(ParseState *pstate, : errmsg("cannot specify %s in JSON mode", "HEADER")); /* Check quote */ - if (opts_out->format != COPY_FORMAT_CSV && opts_out->quote != NULL) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + opts_out->quote != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -905,7 +958,8 @@ ProcessCopyOptions(ParseState *pstate, errmsg("COPY delimiter and quote must be different"))); /* Check escape */ - if (opts_out->format != COPY_FORMAT_CSV && opts_out->escape != NULL) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + opts_out->escape != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -917,7 +971,8 @@ ProcessCopyOptions(ParseState *pstate, errmsg("COPY escape must be a single one-byte character"))); /* Check force_quote */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_quote || opts_out->force_quote_all)) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_quote || opts_out->force_quote_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -931,8 +986,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY FROM"))); /* Check force_notnull */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_notnull != NIL || - opts_out->force_notnull_all)) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_notnull != NIL || opts_out->force_notnull_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -947,8 +1002,8 @@ ProcessCopyOptions(ParseState *pstate, "COPY TO"))); /* Check force_null */ - if (opts_out->format != COPY_FORMAT_CSV && (opts_out->force_null != NIL || - opts_out->force_null_all)) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_CSV && + (opts_out->force_null != NIL || opts_out->force_null_all)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ @@ -995,7 +1050,8 @@ ProcessCopyOptions(ParseState *pstate, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY %s is not supported for %s", "FORMAT JSON", "COPY FROM")); - if (opts_out->format != COPY_FORMAT_JSON && opts_out->force_array) + if (CopyFormatIsBuiltins(opts_out->format) && opts_out->format != COPY_FORMAT_JSON && + opts_out->force_array) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY %s can only be used with JSON mode", "FORCE_ARRAY")); @@ -1048,6 +1104,22 @@ ProcessCopyOptions(ParseState *pstate, * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ errmsg("COPY %s requires %s to be set to %s", "REJECT_LIMIT", "ON_ERROR", "IGNORE"))); + + /* Check custom format routines */ + if (opts_out->format == COPY_FORMAT_CUSTOM) + { + if (is_from && opts_out->custom_format_ent->from_routine == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY format \"%s\" cannot be used with COPY FROM", + opts_out->custom_format_ent->name))); + + if (!is_from && opts_out->custom_format_ent->to_routine == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY format \"%s\" cannot be used with COPY TO", + opts_out->custom_format_ent->name))); + } } /* diff --git a/src/backend/commands/copyapi.c b/src/backend/commands/copyapi.c new file mode 100644 index 00000000000..f007d73e867 --- /dev/null +++ b/src/backend/commands/copyapi.c @@ -0,0 +1,114 @@ +/*------------------------------------------------------------------------- + * + * copyapi.c + * Registry for pluggable COPY TO/FROM format handlers. + * + * The built-in formats (text, csv, binary, json) are dispatched directly by + * the COPY engine. Extensions can provide additional formats by registering + * a CopyToRoutine and/or CopyFromRoutine under a name from their _PG_init(); + * ProcessCopyOptions() then resolves "COPY ... (FORMAT 'name')" against this + * registry. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/commands/copyapi.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/copyapi.h" +#include "utils/memutils.h" + +static CopyCustomFormatEntry *CopyCustomFormatArray = NULL; +static int CopyCustomFormatsAssigned = 0; +static int CopyCustomFormatsAllocated = 0; + +/* Is 'name' one of the built-in format keywords? */ +static bool +is_builtin_copy_format(const char *name) +{ + return (strcmp(name, "text") == 0 || + strcmp(name, "csv") == 0 || + strcmp(name, "binary") == 0 || + strcmp(name, "json") == 0); +} + +/* + * Register a custom COPY format. Intended to be called from an extension's + * _PG_init(). Either routine may be NULL if the format does not support that + * direction (but not both). + * + * 'option_fn' may also be NULL if the format takes no format-specific options. + * + * 'name' is assumed to be a constant string or allocated in storage that will + * never be freed; it is stored by reference. + */ +void +RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, + const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn) +{ + Assert(name != NULL && name[0] != '\0'); + + /* Must support at least one direction */ + Assert(to != NULL || from != NULL); + + Assert(to == NULL || + (to->CopyToStart != NULL && to->CopyToOneRow != NULL && + to->CopyToEnd != NULL)); + Assert(from == NULL || + (from->CopyFromStart != NULL && from->CopyFromOneRow != NULL && + from->CopyFromEnd != NULL)); + + /* Check if it's already used by built-in format names */ + if (is_builtin_copy_format(name)) + elog(ERROR, "COPY format \"%s\" is a built-in format name", name); + + /* Reject a duplicate registration. */ + for (int i = 0; i < CopyCustomFormatsAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].name, name) == 0) + elog(ERROR, "COPY format \"%s\" is already registered", name); + } + + /* Create the array on first use; it must outlive the current context. */ + if (CopyCustomFormatArray == NULL) + { + CopyCustomFormatsAllocated = 16; + CopyCustomFormatArray = (CopyCustomFormatEntry *) + MemoryContextAlloc(TopMemoryContext, + CopyCustomFormatsAllocated * sizeof(CopyCustomFormatEntry)); + } + + /* Expand if full. */ + if (CopyCustomFormatsAssigned >= CopyCustomFormatsAllocated) + { + CopyCustomFormatsAllocated *= 2; + CopyCustomFormatArray = (CopyCustomFormatEntry *) + repalloc_array(CopyCustomFormatArray, CopyCustomFormatEntry, CopyCustomFormatsAllocated); + } + + CopyCustomFormatArray[CopyCustomFormatsAssigned].name = name; + CopyCustomFormatArray[CopyCustomFormatsAssigned].to_routine = to; + CopyCustomFormatArray[CopyCustomFormatsAssigned].from_routine = from; + CopyCustomFormatArray[CopyCustomFormatsAssigned].option_fn = option_fn; + CopyCustomFormatsAssigned++; +} + +/* + * Look up a previously registered custom format. Returns NULL if 'name' is + * not registered. + */ +const CopyCustomFormatEntry * +GetCopyCustomFormatRoutines(const char *name) +{ + for (int i = 0; i < CopyCustomFormatsAssigned; i++) + { + if (strcmp(CopyCustomFormatArray[i].name, name) == 0) + return &(CopyCustomFormatArray[i]); + } + + return NULL; +} diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 2c57b32f4de..ff4fa718e56 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -158,7 +158,9 @@ static const CopyFromRoutine CopyFromRoutineBinary = { static const CopyFromRoutine * CopyFromGetRoutine(const CopyFormatOptions *opts) { - if (opts->format == COPY_FORMAT_CSV) + if (opts->format == COPY_FORMAT_CUSTOM) + return opts->custom_format_ent->from_routine; + else if (opts->format == COPY_FORMAT_CSV) return &CopyFromRoutineCSV; else if (opts->format == COPY_FORMAT_BINARY) return &CopyFromRoutineBinary; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index ef2038c9a5d..72cc1ac8d8d 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -130,7 +130,9 @@ static const CopyToRoutine CopyToRoutineBinary = { static const CopyToRoutine * CopyToGetRoutine(const CopyFormatOptions *opts) { - if (opts->format == COPY_FORMAT_CSV) + if (opts->format == COPY_FORMAT_CUSTOM) + return opts->custom_format_ent->to_routine; + else if (opts->format == COPY_FORMAT_CSV) return &CopyToRoutineCSV; else if (opts->format == COPY_FORMAT_BINARY) return &CopyToRoutineBinary; diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build index 9f258d566eb..d98273da67e 100644 --- a/src/backend/commands/meson.build +++ b/src/backend/commands/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'constraint.c', 'conversioncmds.c', 'copy.c', + 'copyapi.c', 'copyfrom.c', 'copyfromparse.c', 'copyto.c', diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 5e710efff5b..77c652818cb 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -58,7 +58,16 @@ typedef enum CopyFormat COPY_FORMAT_BINARY, COPY_FORMAT_CSV, COPY_FORMAT_JSON, + COPY_FORMAT_CUSTOM, /* format provided by an extension */ } CopyFormat; +#define CopyFormatIsBuiltins(format) ((format) != COPY_FORMAT_CUSTOM) + +/* + * Full definitions live in commands/copyapi.h, which includes this header; + * CopyFormatOptions only needs to hold pointers to the resolved routines. + */ +struct CopyToRoutine; +struct CopyFromRoutine; /* * A struct to hold COPY options, in a parsed form. All of these are related @@ -97,6 +106,15 @@ typedef struct CopyFormatOptions CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ int64 reject_limit; /* maximum tolerable number of errors */ List *convert_select; /* list of column names (can be NIL) */ + + /* + * Resolved handler for a custom format. The directoin not in use may be + * NULL. For built-in formats these are unused. + */ + const struct CopyCustomFormatEntry *custom_format_ent; + + /* Custom format private option data */ + void *format_private_opts; } CopyFormatOptions; /* These are defined in copy_state.h */ diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h index 52cbf5067eb..6c5defbf4ee 100644 --- a/src/include/commands/copy_state.h +++ b/src/include/commands/copy_state.h @@ -178,6 +178,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* Custom format private data to store the state */ + void *format_private; } CopyFromStateData; /* @@ -248,6 +251,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* Custom format private data to store the state */ + void *format_private; } CopyToStateData; #endif /* COPY_STATE_H */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 398e7a78bb3..1b323b23bba 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,6 +14,7 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "commands/copy_state.h" #include "commands/copy.h" /* @@ -102,4 +103,34 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +/* + * Optional callback to process one format-specific COPY option. Invoked + * from ProcessCopyOptions() once per option that core did not recognize, after + * every core option has been parsed (so 'opts' is fully populated). + * + * Returns true if the option belongs to the format and is valid. Returns false + * if the option is not one the format recognizes, in which case core raises the + * "not accepted" error; thus an unrecognized option always errors, whether or + * not the format supplies this callback. For a recognized option with an invalid + * value, the callback should ereport() itself. + */ +typedef bool (*ProcessOneCopyOptionFn) (CopyFormatOptions *opts, bool is_from, + DefElem *option); + +/* + * Sturct to store the registered custom format information. + */ +typedef struct CopyCustomFormatEntry +{ + const char *name; /* constant string; never freed (see below) */ + const CopyToRoutine *to_routine; + const CopyFromRoutine *from_routine; + ProcessOneCopyOptionFn option_fn; +} CopyCustomFormatEntry; + +extern void RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, + const CopyFromRoutine *from, + ProcessOneCopyOptionFn option_fn); +extern const CopyCustomFormatEntry *GetCopyCustomFormatRoutines(const char *name); + #endif /* COPYAPI_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1969d467c1d..5263710e451 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -540,6 +540,7 @@ ConvProcInfo ConversionLocation ConvertRowtypeExpr CookedConstraint +CopyCustomFormatEntry CopyDest CopyFormat CopyFormatOptions -- 2.54.0 [text/x-patch] v3-0004-Add-test-module-for-COPY-custom-format.patch (17.3K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/4-v3-0004-Add-test-module-for-COPY-custom-format.patch) download | inline diff: From b5ff7ebea3141625b8699c85a3bf15972b1e1537 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 13:14:31 -0700 Subject: [PATCH v3 4/4] Add test module for COPY custom format. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + .../modules/test_copy_custom_format/Makefile | 20 +++ .../expected/test_copy_custom_format.out | 105 +++++++++++ .../test_copy_custom_format/meson.build | 32 ++++ .../sql/test_copy_custom_format.sql | 32 ++++ .../test_copy_custom_format.c | 169 ++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 361 insertions(+) create mode 100644 src/test/modules/test_copy_custom_format/Makefile create mode 100644 src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out create mode 100644 src/test/modules/test_copy_custom_format/meson.build create mode 100644 src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql create mode 100644 src/test/modules/test_copy_custom_format/test_copy_custom_format.c diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 0a74ab5c86f..6dcb66174f5 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -23,6 +23,7 @@ SUBDIRS = \ test_cloexec \ test_checksums \ test_copy_callbacks \ + test_copy_custom_format \ test_custom_rmgrs \ test_custom_stats \ test_custom_types \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 4bca42bb370..adfa413fe58 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -23,6 +23,7 @@ subdir('test_bloomfilter') subdir('test_cloexec') subdir('test_checksums') subdir('test_copy_callbacks') +subdir('test_copy_custom_format') subdir('test_cplusplusext') subdir('test_custom_rmgrs') subdir('test_custom_stats') diff --git a/src/test/modules/test_copy_custom_format/Makefile b/src/test/modules/test_copy_custom_format/Makefile new file mode 100644 index 00000000000..68a2a04ff09 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/Makefile @@ -0,0 +1,20 @@ +# src/test/modules/test_copy_custom_format/Makefile + +MODULE_big = test_copy_custom_format +OBJS = \ + $(WIN32RES) \ + test_copy_custom_format.o +PGFILEDESC = "test_copy_custom_format - test custom COPY FORMAT" + +REGRESS = test_copy_custom_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_copy_custom_format +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out new file mode 100644 index 00000000000..817ca3fa60f --- /dev/null +++ b/src/test/modules/test_copy_custom_format/expected/test_copy_custom_format.out @@ -0,0 +1,105 @@ +LOAD 'test_copy_custom_format'; +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789); +COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 2 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToStart: the number of attributes of table: 1, the number of attributes to output: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToOneRow: the number of valid values: 1 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized +ERROR: COPY format "nonexistent" not recognized +LINE 1: COPY copy_data TO stdout WITH (format 'nonexistent'); + ^ +COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting +ERROR: conflicting or redundant options +LINE 1: COPY copy_data TO stdout WITH (format 'text', format 'csv'); + ^ +COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR +ERROR: COPY format "test_format" does not accept option "bogus" +LINE 1: ...PY copy_data TO stdout WITH (format 'test_format', bogus 1); + ^ +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2 +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +ERROR: relation has 3 columns, exceeds max_attributes 2 +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive +ERROR: "max_attributes" must be a positive integer +COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR +ERROR: "max_attributes" must be a positive integer +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required +ERROR: max_attributes requires an integer value +COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate) +ERROR: FREEZE cannot be used with "disallow_freeze" option +COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes of table: 3, the number of attributes to input: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd +-- The built-in options are handled in the same way of built-in formats. +COPY copy_data TO stdout WITH (format 'test_format', delimiter ','); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', quote '"'); +NOTICE: CopyToOutFunc: attribute: smallint +NOTICE: CopyToOutFunc: attribute: integer +NOTICE: CopyToOutFunc: attribute: bigint +NOTICE: CopyToStart: the number of attributes of table: 3, the number of attributes to output: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToEnd +COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO +ERROR: COPY FREEZE cannot be used with COPY TO diff --git a/src/test/modules/test_copy_custom_format/meson.build b/src/test/modules/test_copy_custom_format/meson.build new file mode 100644 index 00000000000..a231ed57649 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/meson.build @@ -0,0 +1,32 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +test_copy_custom_format_sources = files( +'test_copy_custom_format.c', +) + +if host_system == 'windows' + test_copy_custom_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_copy_custom_format', + '--FILEDESC', 'test_copy_custom_format - test custom COPY FORMAT',]) +endif + +test_copy_custom_format = shared_module('test_copy_custom_format', + test_copy_custom_format_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_copy_custom_format + +tests += { + 'name': 'test_copy_custom_format', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_copy_custom_format', + ], + # Disabled because these tests require + # "shared_preload_libraries=test_custom_copy_format", which typical + # runningcheck users do not have (e.g. buildfarm clients). + 'runningcheck': false, + }, +} diff --git a/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql new file mode 100644 index 00000000000..59f58fa55a2 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/sql/test_copy_custom_format.sql @@ -0,0 +1,32 @@ +LOAD 'test_copy_custom_format'; + +CREATE TABLE copy_data (a smallint, b integer, c bigint); +INSERT INTO copy_data VALUES (1,2,3),(12,34,56),(123,456,789); + +COPY copy_data TO stdout WITH (format 'test_format'); -- Start, OutFunc x3, OneRow x3, End +COPY copy_data FROM stdin WITH (format 'test_format'); -- InFunc x3, Start, OneRow, End +\. + +COPY copy_data (a, b) TO stdout WITH (format 'test_format'); -- Start: natts 2 +COPY (SELECT a FROM copy_data) TO stdout WITH (format 'test_format'); -- Start: natts 1 + +COPY copy_data TO stdout WITH (format 'nonexistent'); -- ERROR: not recognized +COPY copy_data TO stdout WITH (format 'text', format 'csv'); -- ERROR: conflicting + +COPY copy_data TO stdout WITH (format 'test_format', bogus 1); -- ERROR + +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 5); -- OK +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 3); -- OK +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 2); -- ERROR: 3 columns exceeds 2 +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 0); -- ERROR: positive +COPY copy_data TO stdout WITH (format 'test_format', max_attributes -1); -- ERROR +COPY copy_data TO stdout WITH (format 'test_format', max_attributes 'x'); -- ERROR: integer required + +COPY copy_data FROM stdin WITH (format 'test_format', freeze true, disallow_freeze true); -- ERROR (validate) +COPY copy_data FROM stdin WITH (format 'test_format', disallow_freeze true); -- OK +\. + +-- The built-in options are handled in the same way of built-in formats. +COPY copy_data TO stdout WITH (format 'test_format', delimiter ','); +COPY copy_data TO stdout WITH (format 'test_format', quote '"'); +COPY copy_data TO stdout WITH (format 'test_format', freeze true); -- ERROR: FREEZE with COPY TO diff --git a/src/test/modules/test_copy_custom_format/test_copy_custom_format.c b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c new file mode 100644 index 00000000000..ca25832fcb0 --- /dev/null +++ b/src/test/modules/test_copy_custom_format/test_copy_custom_format.c @@ -0,0 +1,169 @@ +/*-------------------------------------------------------------------------- + * + * test_copy_custom_format.c + * Code for testing custom COPY format. + * + * Portions Copyright (c) 2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_copy_custom_format/test_copy_custom_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/copyapi.h" +#include "commands/copy_state.h" +#include "commands/defrem.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +typedef struct TestCopyOptions +{ + int max_attributes; + bool disallow_freeze; +} TestCopyOptions; + +static bool +TestCopyProcessOneOption(CopyFormatOptions *opts, bool is_from, DefElem *option) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (t == NULL) + { + t = palloc0_object(TestCopyOptions); + opts->format_private_opts = (void *) t; + } + + if (strcmp(option->defname, "max_attributes") == 0) + { + int val = defGetInt32(option); + + if (val < 1) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"max_attributes\" must be a positive integer")); + + t->max_attributes = val; + return true; + } + else if (strcmp(option->defname, "disallow_freeze") == 0) + { + t->disallow_freeze = defGetBoolean(option); + return true; + } + + return false; +} + +static void +TestCopyValidateOptions(CopyFormatOptions *opts, bool is_from) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (!t) + return; + + if (t->disallow_freeze && opts->freeze) + ereport(ERROR, + errmsg("FREEZE cannot be used with \"disallow_freeze\" option")); +} + +static void +TestCopyFromInFunc(CopyFromState cstate, Oid atttypid, FmgrInfo *finfo, Oid *typioparam) +{ + ereport(NOTICE, + errmsg("CopyFromInFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +check_max_attributes(CopyFormatOptions *opts, TupleDesc tupDesc) +{ + TestCopyOptions *t = (TestCopyOptions *) opts->format_private_opts; + + if (t != NULL && t->max_attributes > 0 && tupDesc->natts > t->max_attributes) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relation has %d columns, exceeds max_attributes %d", + tupDesc->natts, t->max_attributes)); +} + +static void +TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) +{ + check_max_attributes(&cstate->opts, tupDesc); + + ereport(NOTICE, + errmsg("CopyFromStart: the number of attributes of table: %d, the number of attributes to input: %d", + tupDesc->natts, list_length(cstate->attnumlist))); +} + +static bool +TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + ereport(NOTICE, errmsg("CopyFromOneRow")); + + return false; +} + +static void +TestCopyFromEnd(CopyFromState cstate) +{ + ereport(NOTICE, errmsg("CopyFromEnd")); +} + +static void +TestCopyToOutFunc(CopyToState cstate, Oid atttypid, FmgrInfo *finfo) +{ + ereport(NOTICE, errmsg("CopyToOutFunc: attribute: %s", format_type_be(atttypid))); +} + +static void +TestCopyToStart(CopyToState cstate, TupleDesc tupDesc) +{ + check_max_attributes(&cstate->opts, tupDesc); + + ereport(NOTICE, + errmsg("CopyToStart: the number of attributes of table: %d, the number of attributes to output: %d", + tupDesc->natts, list_length(cstate->attnumlist))); +} + +static void +TestCopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: the number of valid values: %u", slot->tts_nvalid))); +} + +static void +TestCopyToEnd(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToEnd"))); +} + +static const CopyToRoutine TestCopyToRoutine = { + .CopyToOutFunc = TestCopyToOutFunc, + .CopyToStart = TestCopyToStart, + .CopyToOneRow = TestCopyToOneRow, + .CopyToEnd = TestCopyToEnd, +}; + + +static const CopyFromRoutine TestCopyFromRoutine = { + .CopyFromInFunc = TestCopyFromInFunc, + .CopyFromStart = TestCopyFromStart, + .CopyFromOneRow = TestCopyFromOneRow, + .CopyFromEnd = TestCopyFromEnd, +}; + +void +_PG_init(void) +{ + RegisterCopyCustomFormat("test_format", + &TestCopyToRoutine, + &TestCopyFromRoutine, + &TestCopyProcessOneOption, + &TestCopyValidateOptions); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5263710e451..552669abc5f 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3179,6 +3179,7 @@ Tcl_Obj Tcl_Size Tcl_Time TempNamespaceStatus +TestCopyOptions TestDSMRegistryHashEntry TestDSMRegistryStruct TestDecodingData -- 2.54.0 [text/x-patch] v3-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch (26.4K, ../../CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com/5-v3-0001-Move-Copy-From-To-StateData-to-copy_state.h.patch) download | inline diff: From 47aa927d65e483331f9b2010499b4c7f0bfb562c Mon Sep 17 00:00:00 2001 From: Masahiko Sawada <[email protected]> Date: Mon, 22 Jun 2026 11:50:27 -0700 Subject: [PATCH v3 1/4] Move Copy[From|To]StateData to copy_state.h. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch-through: --- contrib/file_fdw/file_fdw.c | 2 +- src/backend/commands/copyfrom.c | 5 +- src/backend/commands/copyfromparse.c | 10 +- src/backend/commands/copyto.c | 88 +------- src/include/commands/copy.h | 2 +- src/include/commands/copy_state.h | 253 +++++++++++++++++++++++ src/include/commands/copyfrom_internal.h | 165 +-------------- 7 files changed, 273 insertions(+), 252 deletions(-) create mode 100644 src/include/commands/copy_state.h diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 33a37d832ce..d152d05b92e 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -22,7 +22,7 @@ #include "catalog/pg_authid.h" #include "catalog/pg_foreign_table.h" #include "commands/copy.h" -#include "commands/copyfrom_internal.h" +#include "commands/copy_state.h" #include "commands/defrem.h" #include "commands/explain_format.h" #include "commands/explain_state.h" diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 0087585b2c4..2c57b32f4de 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -30,6 +30,7 @@ #include "access/xact.h" #include "catalog/namespace.h" #include "commands/copyapi.h" +#include "commands/copy_state.h" #include "commands/copyfrom_internal.h" #include "commands/progress.h" #include "commands/trigger.h" @@ -1732,7 +1733,7 @@ BeginCopyFrom(ParseState *pstate, pg_encoding_to_char(GetDatabaseEncoding())))); } - cstate->copy_src = COPY_FILE; /* default */ + cstate->copy_src = COPY_SOURCE_FILE; /* default */ cstate->whereClause = whereClause; @@ -1861,7 +1862,7 @@ BeginCopyFrom(ParseState *pstate, if (data_source_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_src = COPY_CALLBACK; + cstate->copy_src = COPY_SOURCE_CALLBACK; cstate->data_source_cb = data_source_cb; } else if (pipe) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 65fd5a0ab4f..0ff5db4b62d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -184,7 +184,7 @@ ReceiveCopyBegin(CopyFromState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_src = COPY_FRONTEND; + cstate->copy_src = COPY_SOURCE_FRONTEND; cstate->fe_msgbuf = makeStringInfo(); /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); @@ -252,7 +252,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) switch (cstate->copy_src) { - case COPY_FILE: + case COPY_SOURCE_FILE: pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ); bytesread = fread(databuf, 1, maxread, cstate->copy_file); pgstat_report_wait_end(); @@ -263,7 +263,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) if (bytesread == 0) cstate->raw_reached_eof = true; break; - case COPY_FRONTEND: + case COPY_SOURCE_FRONTEND: while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) { int avail; @@ -346,7 +346,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) bytesread += avail; } break; - case COPY_CALLBACK: + case COPY_SOURCE_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); break; } @@ -1259,7 +1259,7 @@ CopyReadLine(CopyFromState cstate, bool is_csv) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (cstate->copy_src == COPY_SOURCE_FRONTEND) { int inbytes; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6755bb698de..ef2038c9a5d 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -23,8 +23,8 @@ #include "access/tupconvert.h" #include "catalog/pg_inherits.h" #include "commands/copyapi.h" +#include "commands/copy_state.h" #include "commands/progress.h" -#include "executor/execdesc.h" #include "executor/executor.h" #include "executor/tuptable.h" #include "funcapi.h" @@ -42,76 +42,6 @@ #include "utils/snapmgr.h" #include "utils/wait_event.h" -/* - * Represents the different dest cases we need to worry about at - * the bottom level - */ -typedef enum CopyDest -{ - COPY_FILE, /* to file (or a piped program) */ - COPY_FRONTEND, /* to frontend */ - COPY_CALLBACK, /* to callback function */ -} CopyDest; - -/* - * This struct contains all the state variables used throughout a COPY TO - * operation. - * - * Multi-byte encodings: all supported client-side encodings encode multi-byte - * characters by having the first byte's high bit set. Subsequent bytes of the - * character can have the high bit not set. When scanning data in such an - * encoding to look for a match to a single-byte (ie ASCII) character, we must - * use the full pg_encoding_mblen() machinery to skip over multibyte - * characters, else we might find a false match to a trailing byte. In - * supported server encodings, there is no possibility of a false match, and - * it's faster to make useless comparisons to trailing bytes than it is to - * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true - * when we have to do it the hard way. - */ -typedef struct CopyToStateData -{ - /* format-specific routines */ - const CopyToRoutine *routine; - - /* low-level state data */ - CopyDest copy_dest; /* type of copy source/destination */ - FILE *copy_file; /* used if copy_dest == COPY_FILE */ - StringInfo fe_msgbuf; /* used for all dests during COPY TO */ - - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - bool json_row_delim_needed; /* need delimiter before next row */ - StringInfo json_buf; /* reusable buffer for JSON output, - * initialized in BeginCopyTo */ - TupleDesc tupDesc; /* Descriptor for JSON output; for a column - * list this is a projected descriptor */ - Datum *json_projvalues; /* pre-allocated projection values, or - * NULL */ - bool *json_projnulls; /* pre-allocated projection nulls, or NULL */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - List *partitions; /* OID list of partitions to copy data from */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - FmgrInfo *out_functions; /* lookup info for output functions */ - MemoryContext rowcontext; /* per-row evaluation context */ - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; - /* DestReceiver for COPY (query) TO */ typedef struct { @@ -559,7 +489,7 @@ SendCopyBegin(CopyToState cstate) } pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->copy_dest = COPY_DEST_FRONTEND; } static void @@ -606,7 +536,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: pgstat_report_wait_start(WAIT_EVENT_COPY_TO_WRITE); if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || @@ -642,11 +572,11 @@ CopySendEndOfRow(CopyToState cstate) } pgstat_report_wait_end(); break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; - case COPY_CALLBACK: + case COPY_DEST_CALLBACK: cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len); break; } @@ -667,7 +597,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) { switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: /* Default line termination depends on platform */ #ifndef WIN32 CopySendChar(cstate, '\n'); @@ -675,7 +605,7 @@ CopySendTextLikeEndOfRow(CopyToState cstate) CopySendString(cstate, "\r\n"); #endif break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); break; @@ -1135,12 +1065,12 @@ BeginCopyTo(ParseState *pstate, /* See Multibyte encoding comment above */ cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - cstate->copy_dest = COPY_FILE; /* default */ + cstate->copy_dest = COPY_DEST_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->copy_dest = COPY_DEST_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index abecfe51098..5e710efff5b 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -99,7 +99,7 @@ typedef struct CopyFormatOptions List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; -/* These are private in commands/copy[from|to].c */ +/* These are defined in copy_state.h */ typedef struct CopyFromStateData *CopyFromState; typedef struct CopyToStateData *CopyToState; diff --git a/src/include/commands/copy_state.h b/src/include/commands/copy_state.h new file mode 100644 index 00000000000..52cbf5067eb --- /dev/null +++ b/src/include/commands/copy_state.h @@ -0,0 +1,253 @@ +/*------------------------------------------------------------------------- + * + * copy_state.h + * prototypes for COPY TO/COPY FROM execution state. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994-5, Regents of the University of California + * + * src/include/commands/copy_state.h + * + *------------------------------------------------------------------------- + */ + +#ifndef COPY_STATE_H +#define COPY_STATE_H + +#include "commands/copy.h" +#include "commands/trigger.h" +#include "executor/execdesc.h" +#include "nodes/miscnodes.h" + +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_SOURCE_FILE, /* from file (or a piped program) */ + COPY_SOURCE_FRONTEND, /* from frontend */ + COPY_SOURCE_CALLBACK, /* from callback function */ +} CopySource; + +/* + * Represents the end-of-line terminator type of the input + */ +typedef enum EolType +{ + EOL_UNKNOWN, + EOL_NL, + EOL_CR, + EOL_CRNL, +} EolType; + +/* + * This struct contains all the state variables used throughout a COPY FROM + * operation. + */ +typedef struct CopyFromStateData +{ + /* format routine */ + const struct CopyFromRoutine *routine; + + /* low-level state data */ + CopySource copy_src; /* type of copy source */ + FILE *copy_file; /* used if copy_src == COPY_SOURCE_FILE */ + StringInfo fe_msgbuf; /* used if copy_src == COPY_SOURCE_FRONTEND */ + + EolType eol_type; /* EOL type of input */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + Oid conversion_proc; /* encoding conversion function */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + CopyFormatOptions opts; + bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ + Node *whereClause; /* WHERE condition (or NULL) */ + + /* these are just for error messages, see CopyFromErrorCallback */ + const char *cur_relname; /* table name for error messages */ + uint64 cur_lineno; /* line number for error messages */ + const char *cur_attname; /* current att for error messages */ + const char *cur_attval; /* current att value for error messages */ + bool relname_only; /* don't output line number, att, etc. */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + AttrNumber num_defaults; /* count of att that are missing and have + * default value */ + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapped during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ + int *defmap; /* array of default att numbers related to + * missing att */ + ExprState **defexprs; /* array of default att expressions for all + * att */ + bool *defaults; /* if DEFAULT marker was found for + * corresponding att */ + bool simd_enabled; /* use SIMD to scan for special chars? */ + + /* + * True if the corresponding attribute's is a constrained domain. This + * will be populated only when ON_ERROR is SET_NULL, otherwise NULL. + */ + bool *domain_with_constraint; + + bool volatile_defexprs; /* is any of defexprs volatile? */ + List *range_table; /* single element list of RangeTblEntry */ + List *rteperminfos; /* single element list of RTEPermissionInfo */ + ExprState *qualexpr; + + TransitionCaptureState *transition_capture; + + /* + * These variables are used to reduce overhead in COPY FROM. + * + * attribute_buf holds the separated, de-escaped text for each field of + * the current line. The CopyReadAttributes functions return arrays of + * pointers into this buffer. We avoid palloc/pfree overhead by re-using + * the buffer on each cycle. + * + * In binary COPY FROM, attribute_buf holds the binary data for the + * current field, but the usage is otherwise similar. + */ + StringInfoData attribute_buf; + + /* field raw data pointers found by COPY FROM */ + + int max_fields; + char **raw_fields; + + /* + * Similarly, line_buf holds the whole input line being processed. The + * input cycle is first to read the whole line into line_buf, and then + * extract the individual attribute fields into attribute_buf. line_buf + * is preserved unmodified so that we can display it in error messages if + * appropriate. (In binary mode, line_buf is not used.) + */ + StringInfoData line_buf; + bool line_buf_valid; /* contains the row being processed? */ + + /* + * input_buf holds input data, already converted to database encoding. + * + * In text mode, CopyReadLine parses this data sufficiently to locate line + * boundaries, then transfers the data to line_buf. We guarantee that + * there is a \0 at input_buf[input_buf_len] at all times. (In binary + * mode, input_buf is not used.) + * + * If encoding conversion is not required, input_buf is not a separate + * buffer but points directly to raw_buf. In that case, input_buf_len + * tracks the number of bytes that have been verified as valid in the + * database encoding, and raw_buf_len is the total number of bytes stored + * in the buffer. + */ +#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ + char *input_buf; + int input_buf_index; /* next byte to process */ + int input_buf_len; /* total # of bytes stored */ + bool input_reached_eof; /* true if we reached EOF */ + bool input_reached_error; /* true if a conversion error happened */ + /* Shorthand for number of unconsumed bytes available in input_buf */ +#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) + + /* + * raw_buf holds raw input data read from the data source (file or client + * connection), not yet converted to the database encoding. Like with + * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. + */ +#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ + char *raw_buf; + int raw_buf_index; /* next byte to process */ + int raw_buf_len; /* total # of bytes stored */ + bool raw_reached_eof; /* true if we reached EOF */ + + /* Shorthand for number of unconsumed bytes available in raw_buf */ +#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) + + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyFromStateData; + +/* + * Represents the different dest cases we need to worry about at + * the bottom level + */ +typedef enum CopyDest +{ + COPY_DEST_FILE, /* to file (or a piped program) */ + COPY_DEST_FRONTEND, /* to frontend */ + COPY_DEST_CALLBACK, /* to callback function */ +} CopyDest; + +/* + * This struct contains all the state variables used throughout a COPY TO + * operation. + * + * Multi-byte encodings: all supported client-side encodings encode multi-byte + * characters by having the first byte's high bit set. Subsequent bytes of the + * character can have the high bit not set. When scanning data in such an + * encoding to look for a match to a single-byte (ie ASCII) character, we must + * use the full pg_encoding_mblen() machinery to skip over multibyte + * characters, else we might find a false match to a trailing byte. In + * supported server encodings, there is no possibility of a false match, and + * it's faster to make useless comparisons to trailing bytes than it is to + * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true + * when we have to do it the hard way. + */ +typedef struct CopyToStateData +{ + /* format-specific routines */ + const struct CopyToRoutine *routine; + + /* low-level state data */ + CopyDest copy_dest; /* type of copy source/destination */ + FILE *copy_file; /* used if copy_dest == COPY_DEST_FILE */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + bool json_row_delim_needed; /* need delimiter before next row */ + StringInfo json_buf; /* reusable buffer for JSON output, + * initialized in BeginCopyTo */ + TupleDesc tupDesc; /* Descriptor for JSON output; for a column + * list this is a projected descriptor */ + Datum *json_projvalues; /* pre-allocated projection values, or + * NULL */ + bool *json_projnulls; /* pre-allocated projection nulls, or NULL */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + List *partitions; /* OID list of partitions to copy data from */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *out_functions; /* lookup info for output functions */ + MemoryContext rowcontext; /* per-row evaluation context */ + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyToStateData; + +#endif /* COPY_STATE_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 9d3e244ee55..f7afade9a39 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -14,31 +14,7 @@ #ifndef COPYFROM_INTERNAL_H #define COPYFROM_INTERNAL_H -#include "commands/copy.h" -#include "commands/trigger.h" -#include "nodes/miscnodes.h" - -/* - * Represents the different source cases we need to worry about at - * the bottom level - */ -typedef enum CopySource -{ - COPY_FILE, /* from file (or a piped program) */ - COPY_FRONTEND, /* from frontend */ - COPY_CALLBACK, /* from callback function */ -} CopySource; - -/* - * Represents the end-of-line terminator type of the input - */ -typedef enum EolType -{ - EOL_UNKNOWN, - EOL_NL, - EOL_CR, - EOL_CRNL, -} EolType; +#include "commands/copy_state.h" /* * Represents the insert method to be used during COPY FROM. @@ -52,145 +28,6 @@ typedef enum CopyInsertMethod * ExecForeignBatchInsert only if valid */ } CopyInsertMethod; -/* - * This struct contains all the state variables used throughout a COPY FROM - * operation. - */ -typedef struct CopyFromStateData -{ - /* format routine */ - const struct CopyFromRoutine *routine; - - /* low-level state data */ - CopySource copy_src; /* type of copy source */ - FILE *copy_file; /* used if copy_src == COPY_FILE */ - StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ - - EolType eol_type; /* EOL type of input */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - Oid conversion_proc; /* encoding conversion function */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; - bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ - Node *whereClause; /* WHERE condition (or NULL) */ - - /* these are just for error messages, see CopyFromErrorCallback */ - const char *cur_relname; /* table name for error messages */ - uint64 cur_lineno; /* line number for error messages */ - const char *cur_attname; /* current att for error messages */ - const char *cur_attval; /* current att value for error messages */ - bool relname_only; /* don't output line number, att, etc. */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - AttrNumber num_defaults; /* count of att that are missing and have - * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ - ErrorSaveContext *escontext; /* soft error trapped during in_functions - * execution */ - uint64 num_errors; /* total number of rows which contained soft - * errors */ - int *defmap; /* array of default att numbers related to - * missing att */ - ExprState **defexprs; /* array of default att expressions for all - * att */ - bool *defaults; /* if DEFAULT marker was found for - * corresponding att */ - bool simd_enabled; /* use SIMD to scan for special chars? */ - - /* - * True if the corresponding attribute's is a constrained domain. This - * will be populated only when ON_ERROR is SET_NULL, otherwise NULL. - */ - bool *domain_with_constraint; - - bool volatile_defexprs; /* is any of defexprs volatile? */ - List *range_table; /* single element list of RangeTblEntry */ - List *rteperminfos; /* single element list of RTEPermissionInfo */ - ExprState *qualexpr; - - TransitionCaptureState *transition_capture; - - /* - * These variables are used to reduce overhead in COPY FROM. - * - * attribute_buf holds the separated, de-escaped text for each field of - * the current line. The CopyReadAttributes functions return arrays of - * pointers into this buffer. We avoid palloc/pfree overhead by re-using - * the buffer on each cycle. - * - * In binary COPY FROM, attribute_buf holds the binary data for the - * current field, but the usage is otherwise similar. - */ - StringInfoData attribute_buf; - - /* field raw data pointers found by COPY FROM */ - - int max_fields; - char **raw_fields; - - /* - * Similarly, line_buf holds the whole input line being processed. The - * input cycle is first to read the whole line into line_buf, and then - * extract the individual attribute fields into attribute_buf. line_buf - * is preserved unmodified so that we can display it in error messages if - * appropriate. (In binary mode, line_buf is not used.) - */ - StringInfoData line_buf; - bool line_buf_valid; /* contains the row being processed? */ - - /* - * input_buf holds input data, already converted to database encoding. - * - * In text mode, CopyReadLine parses this data sufficiently to locate line - * boundaries, then transfers the data to line_buf. We guarantee that - * there is a \0 at input_buf[input_buf_len] at all times. (In binary - * mode, input_buf is not used.) - * - * If encoding conversion is not required, input_buf is not a separate - * buffer but points directly to raw_buf. In that case, input_buf_len - * tracks the number of bytes that have been verified as valid in the - * database encoding, and raw_buf_len is the total number of bytes stored - * in the buffer. - */ -#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ - char *input_buf; - int input_buf_index; /* next byte to process */ - int input_buf_len; /* total # of bytes stored */ - bool input_reached_eof; /* true if we reached EOF */ - bool input_reached_error; /* true if a conversion error happened */ - /* Shorthand for number of unconsumed bytes available in input_buf */ -#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) - - /* - * raw_buf holds raw input data read from the data source (file or client - * connection), not yet converted to the database encoding. Like with - * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. - */ -#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ - char *raw_buf; - int raw_buf_index; /* next byte to process */ - int raw_buf_len; /* total # of bytes stored */ - bool raw_reached_eof; /* true if we reached EOF */ - - /* Shorthand for number of unconsumed bytes available in raw_buf */ -#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) - - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; - extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); -- 2.54.0 ^ permalink raw reply [nested|flat] 127+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2026-06-26 03:21 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 127+ messages in thread From: Sutou Kouhei @ 2026-06-26 03:21 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBLbmxT6xMHkDjfvBZEJUo1VbCd-vWhwX9WXbpMgaGW_A@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 23 Jun 2026 18:15:10 -0700, Masahiko Sawada <[email protected]> wrote: > Thank you for reviewing the patches! I've attached updated patches. +1 I have only a few minor comments: 0002: > --- /dev/null > +++ b/src/backend/commands/copyapi.c > +void > +RegisterCopyCustomFormat(const char *name, const CopyToRoutine *to, > + const CopyFromRoutine *from, ProcessOneCopyOptionFn option_fn) How about using "const CopyCustomFormatEntry *" instead of "to", "from" and "option_fn"? If we use CopyCustomFormatEntry here, we don't need change the signature of this function when we add more items. > +const CopyCustomFormatEntry * > +GetCopyCustomFormatRoutines(const char *name) How about renaming this to "...FormatEntry" from "...FormatRoutines"? > --- a/src/include/commands/copy.h > +++ b/src/include/commands/copy.h > +#define CopyFormatIsBuiltins(format) ((format) != COPY_FORMAT_CUSTOM) How about removing the last "s" ("...IsBuiltin") because this processes only one format? > + const struct CopyCustomFormatEntry *custom_format_ent; It may be better that we don't abbreviate "_entry" to "_ent" here for readability. It seems that we use this abbreviation only in a few places: $ git grep '_ent;' src/ src/backend/replication/logical/reorderbuffer.c: ReorderBufferTupleCidEnt *new_ent; src/backend/utils/cache/catcache.c: CatCInProgress in_progress_ent; src/backend/utils/cache/catcache.c: catcache_in_progress_stack = &in_progress_ent; src/backend/utils/cache/catcache.c: CatCInProgress in_progress_ent; src/backend/utils/cache/catcache.c: catcache_in_progress_stack = &in_progress_ent; > I'll verify that the new API works well with an experimental custom > copy format extension. I think that we need to provide more APIs to read/write data like we did in v40-0003 to implement a custom copy format extension: https://www.postgresql.org/message-id/flat/20250425.214534.1841428689427124725.kou%40clear-code.com At least https://github.com/kou/pg-copy-arrow needs them. Thanks, -- kou ^ permalink raw reply [nested|flat] 127+ messages in thread
end of thread, other threads:[~2026-06-26 03:21 UTC | newest] Thread overview: 127+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-07-22 15:09 [PATCH v2] Check partitions not in use Alvaro Herrera <[email protected]> 2023-02-16 06:06 [PATCH v20 5/5] introduce restore_library Nathan Bossart <[email protected]> 2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]> 2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]> 2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-17 20:50 Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-19 02:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-20 00:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-20 01:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-23 09:01 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-27 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-29 05:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-29 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-29 08:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-29 16:12 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-31 19:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-02 22:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-03 02:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-03 02:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-03 04:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-03 04:56 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-03 06:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-03 06:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-09 08:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-10 04:29 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-26 01:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-03 05:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-05-03 05:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-03 06:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-05-03 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-03 14:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-05-04 05:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-04 05:27 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-05-09 09:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-05-10 00:57 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-05-26 01:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-12 02:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2025-06-12 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-06-16 23:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-17 00:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2025-06-18 03:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-24 02:59 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-06-24 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-24 06:24 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-06-24 07:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-24 15:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-06-25 07:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-06-30 06:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-07-13 18:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-07-14 08:38 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-07-15 07:54 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-07-17 20:44 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-07-18 10:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-07-28 19:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-07-29 05:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-08-14 06:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-09-08 21:08 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-09-09 02:50 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-09-09 20:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-09-10 02:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-09-10 07:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-09-11 05:46 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-09-11 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-09-12 00:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-09-15 17:00 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-10-03 07:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-10-13 21:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-10-14 02:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-10-29 20:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-11-14 20:19 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-11-17 06:40 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-11-17 17:04 ` Re: Make COPY format extendable: Extract COPY TO format implementations Tomas Vondra <[email protected]> 2025-12-02 02:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-12-18 23:43 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2026-06-23 01:06 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2026-06-23 05:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2026-06-24 01:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2026-06-26 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-07-15 12:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andres Freund <[email protected]> 2025-07-17 20:33 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-07-18 09:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-29 16:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-30 02:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2025-03-31 18:51 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-31 20:42 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-31 17:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-04-06 16:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-25 00:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-22 00:31 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-22 05:07 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2025-03-22 05:23 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]> 2025-03-22 17:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations David G. Johnston <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox