public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v20 5/5] introduce restore_library 38+ messages / 7 participants [nested] [flat]
* [PATCH v20 5/5] introduce restore_library @ 2023-02-16 06:06 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 38+ messages in thread From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw) --- contrib/basic_archive/Makefile | 2 + contrib/basic_archive/basic_archive.c | 153 +++++++- contrib/basic_archive/meson.build | 5 + contrib/basic_archive/t/001_restore.pl | 44 +++ doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++- doc/src/sgml/backup.sgml | 40 +- doc/src/sgml/basic-archive.sgml | 31 +- doc/src/sgml/config.sgml | 53 ++- doc/src/sgml/high-availability.sgml | 18 +- src/backend/access/transam/xlog.c | 20 +- src/backend/access/transam/xlogarchive.c | 96 ++++- src/backend/access/transam/xlogrecovery.c | 14 +- src/backend/postmaster/checkpointer.c | 54 +++ src/backend/postmaster/startup.c | 44 ++- src/backend/restore/shell_restore.c | 85 ++++- src/backend/utils/misc/guc_tables.c | 11 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/restore/restore_module.h | 143 +++++++ src/include/restore/shell_restore.h | 16 +- src/tools/pgindent/typedefs.list | 2 + 20 files changed, 1104 insertions(+), 84 deletions(-) create mode 100644 contrib/basic_archive/t/001_restore.pl create mode 100644 src/include/restore/restore_module.h diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile index 100ed81f12..e61f7d676f 100644 --- a/contrib/basic_archive/Makefile +++ b/contrib/basic_archive/Makefile @@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c # typical installcheck users do not have (e.g. buildfarm clients). NO_INSTALLCHECK = 1 +TAP_TESTS = 1 + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c index 6b102e9072..e0d8dce364 100644 --- a/contrib/basic_archive/basic_archive.c +++ b/contrib/basic_archive/basic_archive.c @@ -17,6 +17,11 @@ * a file is successfully archived and then the system crashes before * a durable record of the success has been made. * + * This file also demonstrates a basic restore library implementation that + * is roughly equivalent to the following shell command: + * + * cp /path/to/archivedir/%f %p + * * Copyright (c) 2022-2024, PostgreSQL Global Development Group * * IDENTIFICATION @@ -33,6 +38,7 @@ #include "archive/archive_module.h" #include "common/int.h" #include "miscadmin.h" +#include "restore/restore_module.h" #include "storage/copydir.h" #include "storage/fd.h" #include "utils/guc.h" @@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path); static bool check_archive_directory(char **newval, void **extra, GucSource source); static bool compare_files(const char *file1, const char *file2); static void basic_archive_shutdown(ArchiveModuleState *state); +static bool basic_restore_configured(RestoreModuleState *state); +static bool basic_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +static bool basic_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path); +static bool basic_restore_file(const char *file, const char *path); +static bool basic_restore_timeline_history_exists(RestoreModuleState *state, + const char *file, + const char *path); static const ArchiveModuleCallbacks basic_archive_callbacks = { .startup_cb = basic_archive_startup, @@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = { .shutdown_cb = basic_archive_shutdown }; +static const RestoreModuleCallbacks basic_restore_callbacks = { + .startup_cb = NULL, + .restore_wal_segment_configured_cb = basic_restore_configured, + .restore_wal_segment_cb = basic_restore_wal_segment, + .restore_timeline_history_configured_cb = basic_restore_configured, + .restore_timeline_history_cb = basic_restore_timeline_history, + .timeline_history_exists_configured_cb = basic_restore_configured, + .timeline_history_exists_cb = basic_restore_timeline_history_exists, + .archive_cleanup_configured_cb = NULL, + .archive_cleanup_cb = NULL, + .recovery_end_configured_cb = NULL, + .recovery_end_cb = NULL, + .shutdown_cb = NULL +}; + /* * _PG_init * @@ -71,7 +102,7 @@ void _PG_init(void) { DefineCustomStringVariable("basic_archive.archive_directory", - gettext_noop("Archive file destination directory."), + gettext_noop("Archive file source/destination directory."), NULL, &archive_directory, "", @@ -93,6 +124,17 @@ _PG_archive_module_init(void) return &basic_archive_callbacks; } +/* + * _PG_restore_module_init + * + * Returns the module's restore callbacks. + */ +const RestoreModuleCallbacks * +_PG_restore_module_init(void) +{ + return &basic_restore_callbacks; +} + /* * basic_archive_startup * @@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state) pfree(data); state->private_data = NULL; } + +/* + * basic_restore_configured + * + * Checks that archive_directory is not blank. + */ +static bool +basic_restore_configured(RestoreModuleState *state) +{ + return archive_directory != NULL && archive_directory[0] != '\0'; +} + +/* + * basic_restore_wal_segment + * + * Retrieves one archived WAL segment. + */ +static bool +basic_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName) +{ + return basic_restore_file(file, path); +} + +/* + * basic_restore_timeline_history + * + * Retrieves one timeline history file. + */ +static bool +basic_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path) +{ + return basic_restore_file(file, path); +} + +/* + * basic_restore_file + * + * Retrieves one file. + */ +static bool +basic_restore_file(const char *file, const char *path) +{ + char archive_path[MAXPGPATH]; + struct stat st; + + ereport(DEBUG1, + (errmsg("restoring \"%s\" via basic_archive", + file))); + + /* + * If the file doesn't exist, return false to indicate that there are no + * more files to restore. + */ + snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file); + if (stat(archive_path, &st) != 0) + { + int elevel = (errno == ENOENT) ? DEBUG1 : ERROR; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + archive_path))); + return false; + } + + copy_file(archive_path, path); + + ereport(DEBUG1, + (errmsg("restored \"%s\" via basic_archive", + file))); + return true; +} + +/* + * basic_restore_timeline_history_exists + * + * Check whether a timeline history file is present in the archive directory. + */ +static bool +basic_restore_timeline_history_exists(RestoreModuleState *state, + const char *file, const char *path) +{ + char archive_path[MAXPGPATH]; + struct stat st; + + ereport(DEBUG1, + (errmsg("checking existence of \"%s\" via basic_archive", + file))); + + snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file); + if (stat(archive_path, &st) != 0) + { + int elevel = (errno == ENOENT) ? DEBUG1 : ERROR; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + archive_path))); + return false; + } + + ereport(DEBUG1, + (errmsg("verified existence of \"%s\" via basic_archive", + file))); + return true; +} diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build index cc2f62bf36..8e36fa7ed6 100644 --- a/contrib/basic_archive/meson.build +++ b/contrib/basic_archive/meson.build @@ -31,4 +31,9 @@ tests += { # typical installcheck users do not have (e.g. buildfarm clients). 'runningcheck': false, }, + 'tap': { + 'tests': [ + 't/001_restore.pl', + ], + }, } diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl new file mode 100644 index 0000000000..6c01f736cf --- /dev/null +++ b/contrib/basic_archive/t/001_restore.pl @@ -0,0 +1,44 @@ + +# Copyright (c) 2024, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# start a node +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(has_archiving => 1, allows_streaming => 1); +my $archive_dir = $node->archive_dir; +$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +$node->append_conf('postgresql.conf', "archive_command = ''"); +$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'"); +$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$node->start; + +# backup the node +my $backup = 'backup'; +$node->backup($backup); + +# generate some new WAL files +$node->safe_psql('postgres', "CREATE TABLE test (a INT);"); +$node->safe_psql('postgres', "SELECT pg_switch_wal();"); +$node->safe_psql('postgres', "INSERT INTO test VALUES (1);"); + +# shut down the node (this should archive all WAL files) +$node->stop; + +# restore from the backup +my $restore = PostgreSQL::Test::Cluster->new('restore'); +$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0); +$restore->append_conf('postgresql.conf', "restore_command = ''"); +$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'"); +$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$restore->start; + +# ensure post-backup WAL is replayed +my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;"); +is($result, "1", "check restore content"); + +done_testing(); diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml index a52d15082d..15be548a73 100644 --- a/doc/src/sgml/archive-and-restore-modules.sgml +++ b/doc/src/sgml/archive-and-restore-modules.sgml @@ -8,15 +8,17 @@ <para> PostgreSQL provides infrastructure to create custom modules for continuous - archiving (see <xref linkend="continuous-archiving"/>). While archiving via - a shell command (i.e., <xref linkend="guc-archive-command"/>) is much - simpler, a custom archive module will often be considerably more robust and - performant. + archiving and recovery (see <xref linkend="continuous-archiving"/>). While a + shell command (i.e., <xref linkend="guc-archive-command"/>, + <xref linkend="guc-restore-command"/>) is much simpler, a custom module will + often be considerably more robust and performant. </para> <para> The <filename>contrib/basic_archive</filename> module contains a working - example, which demonstrates some useful techniques. + example, which demonstrates some useful techniques. As demonstrated in + <filename>basic_archive</filename> a single module may serve as both an + archive module and a restore module. </para> <sect1 id="archive-modules"> @@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state); </sect3> </sect2> </sect1> + + <sect1 id="restore-modules"> + <title>Restore Modules</title> + <indexterm zone="restore-modules"> + <primary>Restore Modules</primary> + </indexterm> + + <para> + When a custom <xref linkend="guc-restore-library"/> is configured, + PostgreSQL will use the module for restore actions. It is + ultimately up to the module to decide how to accomplish each task, + but some recommendations are listed at + <xref linkend="backup-pitr-recovery"/>. + </para> + + <para> + Restore modules must at least consist of an initialization function + (see <xref linkend="restore-module-init"/>) and the required callbacks (see + <xref linkend="restore-module-callbacks"/>). However, restore modules are + also permitted to do much more (e.g., declare GUCs and register background + workers). + </para> + + <sect2 id="restore-module-init"> + <title>Initialization Functions</title> + <indexterm zone="restore-module-init"> + <primary>_PG_restore_module_init</primary> + </indexterm> + + <para> + An restore library is loaded by dynamically loading a shared library with + the <xref linkend="guc-restore-library"/>'s name as the library base name. + The normal library search path is used to locate the library. To provide + the required restore module callbacks and to indicate that the library is + actually a restore module, it needs to provide a function named + <function>_PG_restore_module_init</function>. The result of the function + must be a pointer to a struct of type + <structname>RestoreModuleCallbacks</structname>, which contains everything + that the core code needs to know how to make use of the restore module. + The return value needs to be of server lifetime, which is typically + achieved by defining it as a <literal>static const</literal> variable in + global scope. + +<programlisting> +typedef struct RestoreModuleCallbacks +{ + RestoreStartupCB startup_cb; + RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb; + RestoreWalSegmentCB restore_wal_segment_cb; + RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb; + RestoreTimelineHistoryCB restore_timeline_history_cb; + TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb; + TimelineHistoryExistsCB timeline_history_exists_cb; + ArchiveCleanupConfiguredCB archive_cleanup_configured_cb; + ArchiveCleanupCB archive_cleanup_cb; + RecoveryEndConfiguredCB recovery_end_configured_cb; + RecoveryEndCB recovery_end_cb; + RestoreShutdownCB shutdown_cb; +} RestoreModuleCallbacks; +typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void); +</programlisting> + + The <function>restore_wal_segment_configured_cb</function>, + <function>restore_wal_segment_cb</function>, + <function>restore_timeline_history_configured_cb</function>, + <function>restore_timeline_history_cb</function>, + <function>timeline_history_exists_configured_cb</function>, and + <function>timeline_history_exists_cb</function> callbacks are required for + archive recovery but optional for streaming replication. The others are + always optional. + </para> + </sect2> + + <sect2 id="restore-module-callbacks"> + <title>Restore Module Callbacks</title> + + <para> + The restore callbacks define the actual behavior of the module. The server + will call them as required to execute restore actions. + </para> + + <sect3 id="restore-module-startup"> + <title>Startup Callback</title> + + <para> + The <function>startup_cb</function> callback is called shortly after the + module is loaded. This callback can be used to perform any additional + initialization required. If the restore module has state, it can use + <structfield>state->private_data</structfield> to store it. + +<programlisting> +typedef void (*RestoreStartupCB) (RestoreModuleState *state); +</programlisting> + </para> + </sect3> + + <sect3 id="restore-module-restore-wal-segment-configured"> + <title>Restore WAL Segment Configured Callback</title> + <para> + The <function>restore_wal_segment_configured_cb</function> callback is + called to determine whether the module is fully configured and ready to + accept requests to retrieve WAL files (e.g., its configuration parameters + are set to valid values). If no + <function>restore_wal_segment_configured_cb</function> is defined, the + server always assumes the module is not configured to retrieve WAL files. + +<programlisting> +typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will retrieve WAL files + by calling the <function>restore_wal_segment_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>restore_wal_segment_cb</function> callback to retrieve WAL + files. + </para> + </sect3> + + <sect3 id="restore-module-restore-wal-segment"> + <title>Restore WAL Segment Callback</title> + <para> + The <function>restore_wal_segment_cb</function> callback is called to + retrieve a single archived segment of the WAL file series for archive + recovery or streaming replication. + +<programlisting> +typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName); +</programlisting> + + This callback must return <literal>true</literal> only if the file was + successfully retrieved. If the file is not available in the archives, the + callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the WAL file to retrieve, while <replaceable>path</replaceable> + contains the destination's relative path (including the file name). + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point. That is the earliest + file that must be kept to allow a restore to be restartable, so this + information can be used to truncate the archive to just the minimum + required to support restarting from the current restore. + <replaceable>lastRestartPointFileName</replaceable> is typically only used + by warm-standby configurations (see <xref linkend="warm-standby"/>). Note + that if multiple standby servers are restoring from the same archive + directory, you will need to ensure that you do not delete WAL files until + they are no longer needed by any of the servers. + </para> + </sect3> + + <sect3 id="restore-module-restore-timeline-history-configured"> + <title>Restore Timeline History Configured Callback</title> + <para> + The <function>restore_timeline_history_configured_cb</function> callback + is called to determine whether the module is fully configured and ready to + accept requests to retrieve timeline history files (e.g., its + configuration parameters are set to valid values). If no + <function>restore_timeline_history_configured_cb</function> is defined, + the server always assumes the module is not configured to retrieve + timeline history files. + +<programlisting> +typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will retrieve timeline + history files by calling the + <function>restore_timeline_history_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>restore_timeline_history_cb</function> callback to retrieve + timeline history files. + </para> + </sect3> + + <sect3 id="restore-module-restore-timeline-history"> + <title>Restore Timeline History Callback</title> + <para> + The <function>restore_timeline_history_cb</function> callback is called to + retrieve a single archived timeline history file for archive recovery or + streaming replication. + +<programlisting> +typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path); +</programlisting> + + This callback must return <literal>true</literal> only if the file was + successfully retrieved. If the file is not available in the archives, the + callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the WAL file to retrieve, while <replaceable>path</replaceable> + contains the destination's relative path (including the file name). + </para> + </sect3> + + <sect3 id="restore-module-timeline-history-exists-configured"> + <title>Timeline History Exists Configured Callback</title> + <para> + The <function>timeline_history_exists_configured_cb</function> callback is + called to determine whether the module is fully configured and ready to + accept requests to determine whether a timeline history file exists in the + archives (e.g., its configuration parameters are set to valid values). If + no <function>timeline_history_exists_configured_cb</function> is defined, + the server always assumes the module is not configured to check whether + certain timeline history files exist. + +<programlisting> +typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will check whether + certain timeline history files are present in the archives by calling the + <function>timeline_history_exists_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>timeline_history_exists_cb</function> callback to check if + timeline history files exist in the archives. + </para> + </sect3> + + <sect3 id="restore-module-timeline-history-exists"> + <title>Timeline History Exists Callback</title> + <para> + The <function>timeline_history_exists_cb</function> callback is called to + check whether a timeline history file is present in the archives. + +<programlisting> +typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path); +</programlisting> + + This callback must return <literal>true</literal> only if the file is + present in the archives. If the file is not available in the archives, + the callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the timeline history file. + </para> + + <para> + Some restore modules might not have a dedicated way to determine whether a + timeline history file exists. For example, + <varname>restore_command</varname> only tells the server how to retrieve + files. In this case, the <function>timeline_history_exists_cb</function> + callback should copy the file to <replaceable>path</replaceable>, which + contains the destination's relative path (including the file name), and + the restore module should set + <structfield>state->timeline_history_exists_cb_copies</structfield> to + <literal>true</literal>. It is recommended to set this flag in the + module's <function>startup_cb</function> callback. This flag tells the + server that it should verify that the file was successfully retrieved + after invoking this callback. + </para> + </sect3> + + <sect3 id="restore-module-archive-cleanup-configured"> + <title>Archive Cleanup Configured Callback</title> + <para> + The <function>archive_cleanup_configured_cb</function> callback is called + to determine whether the module is fully configured and ready to accept + requests to remove old WAL files from the archives (e.g., its + configuration parameters are set to valid values). If no + <function>archive_cleanup_configured_cb</function> is defined, the server + always assumes the module is not configured to remove old WAL files. + +<programlisting> +typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will remove old WAL + files by calling the <function>archive_cleanup_cb</function> callback. If + <literal>false</literal> is returned, the server will not use the + <function>archive_cleanup_cb</function> callback to remove old WAL files. + </para> + </sect3> + + <sect3 id="restore-module-archive-cleanup"> + <title>Archive Cleanup Callback</title> + <para> + The <function>archive_cleanup_cb</function> callback is called at every + restart point by the checkpointer process and is intended to provide a + mechanism for cleaning up old archived WAL files that are no longer + needed by the standby server. + +<programlisting> +typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the + name of the file that includes the last valid restart point, like in + <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>. + </para> + </sect3> + + <sect3 id="restore-module-recovery-end-configured"> + <title>Recovery End Configured Callback</title> + <para> + The <function>recovery_end_configured_cb</function> callback is called to + determine whether the module is fully configured and ready to execute its + <function>recovery_end_cb</function> callback once at the end of recovery + (e.g., its configuration parameters are set to valid values). If no + <function>recovery_end_configured_cb</function> is defined, the server + always assumes the module is not configured to execute its + <function>recovery_end_cb</function> callback. + +<programlisting> +typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state); +</programlisting> + + If <literal>true</literal> is returned, the server will execute the + <function>recovery_end_cb</function> callback once at the end of recovery. + If <literal>false</literal> is returned, the server will not use the + <function>recovery_end_cb</function> callback at the end of recovery. + </para> + </sect3> + + <sect3 id="restore-module-recovery-end"> + <title>Recovery End Callback</title> + <para> + The <function>recovery_end_cb</function> callback is called once at the + end of recovery and is intended to provide a mechanism for cleanup + following the end of replication or recovery. + +<programlisting> +typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point, like in + <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>. + </para> + </sect3> + + <sect3 id="restore-module-shutdown"> + <title>Shutdown Callback</title> + <para> + The <function>shutdown_cb</function> callback is called when a process + that has loaded the restore module exits (e.g., after an error) or the + value of <xref linkend="guc-restore-library"/> changes. If no + <function>shutdown_cb</function> is defined, no special action is taken in + these situations. If the restore module has state, this callback should + free it to avoid leaks. + +<programlisting> +typedef void (*RestoreShutdownCB) (RestoreModuleState *state); + </programlisting> + </para> + </sect3> + </sect2> + </sect1> </chapter> diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index b3468eea3c..e1643f92e0 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true); <para> The key part of all this is to set up a recovery configuration that describes how you want to recover and how far the recovery should - run. The one thing that you absolutely must specify is the <varname>restore_command</varname>, + run. The one thing that you absolutely must specify is either + <varname>restore_command</varname> or a <varname>restore_library</varname>, which tells <productname>PostgreSQL</productname> how to retrieve archived - WAL file segments. Like the <varname>archive_command</varname>, this is + WAL file segments. + </para> + + <para> + Like the <varname>archive_library</varname> parameter, + <varname>restore_library</varname> is a shared library. Since such + libraries are written in <literal>C</literal>, creating your own may + require considerably more effort than writing a shell command. However, + restore modules can be more performance than restoring via shell, and they + will have access to many useful server resources. For more information + about creating a <varname>restore_library</varname>, see + <xref linkend="restore-modules"/>. + </para> + + <para> + Like the <varname>archive_command</varname>, + <varname>restore_command</varname> is a shell command string. It can contain <literal>%f</literal>, which is replaced by the name of the desired WAL file, and <literal>%p</literal>, which is replaced by the path name to copy the WAL file to. @@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' </para> <para> - It is important that the command return nonzero exit status on failure. - The command <emphasis>will</emphasis> be called requesting files that are not - present in the archive; it must return nonzero when so asked. This is not - an error condition. An exception is that if the command was terminated by + It is important that the <varname>restore_command</varname> return nonzero + exit status on failure, or, if you are using a + <varname>restore_library</varname>, that the restore callbacks return + <literal>false</literal> on failure. The command or library + <emphasis>will</emphasis> be called requesting files that are not + present in the archive; it must fail when so asked. This is not + an error condition. An exception is that if the + <varname>restore_command</varname> was terminated by a signal (other than <systemitem>SIGTERM</systemitem>, which is used as part of a database server shutdown) or an error by the shell (such as command not found), then recovery will abort and the server will not start - up. + up. Likewise, if the restore callbacks provided by the + <varname>restore_library</varname> emit an <literal>ERROR</literal> or + <literal>FATAL</literal>, recovery will abort and the server won't start. </para> <para> @@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' close as possible given the available WAL segments). Therefore, a normal recovery will end with a <quote>file not found</quote> message, the exact text of the error message depending upon your choice of - <varname>restore_command</varname>. You may also see an error message + <varname>restore_command</varname> or <varname>restore_library</varname>. + You may also see an error message at the start of recovery for a file named something like <filename>00000001.history</filename>. This is also normal and does not indicate a problem in simple recovery situations; see diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml index b4d43ced20..f3a5a502bd 100644 --- a/doc/src/sgml/basic-archive.sgml +++ b/doc/src/sgml/basic-archive.sgml @@ -8,17 +8,20 @@ </indexterm> <para> - <filename>basic_archive</filename> is an example of an archive module. This - module copies completed WAL segment files to the specified directory. This - may not be especially useful, but it can serve as a starting point for - developing your own archive module. For more information about archive - modules, see <xref linkend="archive-modules"/>. + <filename>basic_archive</filename> is an example of an archive and restore + module. This module copies completed WAL segment files to or from the + specified directory. This may not be especially useful, but it can serve as + a starting point for developing your own archive and restore modules. For + more information about archive and restore modules, see\ + <xref linkend="archive-and-restore-modules"/>. </para> <para> - In order to function, this module must be loaded via + For use as an archive module, this module must be loaded via <xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/> - must be enabled. + must be enabled. For use as a restore module, this module must be loaded + via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see + <xref linkend="runtime-config-wal-archive-recovery"/>). </para> <sect2 id="basic-archive-configuration-parameters"> @@ -34,11 +37,12 @@ </term> <listitem> <para> - The directory where the server should copy WAL segment files. This - directory must already exist. The default is an empty string, which - effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/> - is enabled, the server will accumulate WAL segment files in the - expectation that a value will soon be provided. + The directory where the server should copy WAL segment files to or from. + This directory must already exist. The default is an empty string, + which, when used for archiving, effectively halts WAL archival, but if + <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate + WAL segment files in the expectation that a value will soon be provided. + When an empty string is used for recovery, restore will fail. </para> </listitem> </varlistentry> @@ -46,7 +50,7 @@ <para> These parameters must be set in <filename>postgresql.conf</filename>. - Typical usage might be: + Typical usage as an archive module might be: </para> <programlisting> @@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory' <title>Notes</title> <para> + When <filename>basic_archive</filename> is used as an archive module, Server crashes may leave temporary files with the prefix <filename>archtemp</filename> in the archive directory. It is recommended to delete such files before restarting the server after a crash. It is safe to diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 65a6e6c408..9a2570f87f 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3906,7 +3906,8 @@ include_dir 'conf.d' recovery when the end of archived WAL is reached, but will keep trying to continue recovery by connecting to the sending server as specified by the <varname>primary_conninfo</varname> setting and/or by fetching new WAL - segments using <varname>restore_command</varname>. For this mode, the + segments using <varname>restore_command</varname> or + <varname>restore_library</varname>. For this mode, the parameters from this section and <xref linkend="runtime-config-replication-standby"/> are of interest. Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will @@ -3934,7 +3935,8 @@ include_dir 'conf.d' <listitem> <para> The local shell command to execute to retrieve an archived segment of - the WAL file series. This parameter is required for archive recovery, + the WAL file series. Either <varname>restore_command</varname> or + <xref linkend="guc-restore-library"/> is required for archive recovery, but optional for streaming replication. Any <literal>%f</literal> in the string is replaced by the name of the file to retrieve from the archive, @@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>restore_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-restore-library" xreflabel="restore_library"> + <term><varname>restore_library</varname> (<type>string</type>) + <indexterm> + <primary><varname>restore_library</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + The library to use for restore actions, including retrieving archived + files and executing tasks at restartpoints and at recovery end. Either + <xref linkend="guc-restore-command"/> or + <varname>restore_library</varname> is required for archive recovery, + but optional for streaming replication. If this parameter is set to an + empty string (the default), restoring via shell is enabled, and + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname>, and + <varname>recovery_end_command</varname> are used. If both + <varname>restore_library</varname> and any of + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname>, or + <varname>recovery_end_command</varname> are set, an error will be + raised. Otherwise, the specified shared library is used for restoring. + For more information, see <xref linkend="restore-modules"/>. + </para> + + <para> + This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. </para> </listitem> </varlistentry> @@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>archive_cleanup_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> @@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>recovery_end_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index b48209fc2f..a86a9fe93a 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order. <para> In standby mode, the server continuously applies WAL received from the primary server. The standby server can read WAL from a WAL archive - (see <xref linkend="guc-restore-command"/>) or directly from the primary + (see <xref linkend="guc-restore-command"/> and + <xref linkend="guc-restore-library"/>) or directly from the primary over a TCP connection (streaming replication). The standby server will also attempt to restore any WAL found in the standby cluster's <filename>pg_wal</filename> directory. That typically happens after a server @@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order. <para> At startup, the standby begins by restoring all WAL available in the - archive location, calling <varname>restore_command</varname>. Once it + archive location, either by calling <varname>restore_command</varname> or + by executing the <varname>restore_library</varname>'s callbacks. Once it reaches the end of WAL available there and <varname>restore_command</varname> + or <varname>restore_library</varname> fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory. If that fails, and streaming replication has been configured, the standby tries to connect to the primary server and start streaming WAL @@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order. server (see <xref linkend="backup-pitr-recovery"/>). Create a file <link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm> in the standby's cluster data - directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from + directory. Set <xref linkend="guc-restore-command"/> or + <xref linkend="guc-restore-library"/> to copy files from the WAL archive. If you plan to have multiple standby servers for high availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to <literal>latest</literal> (the default), to make the standby server follow the timeline change @@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order. <note> <para> - <xref linkend="guc-restore-command"/> should return immediately + <xref linkend="guc-restore-command"/> and + <xref linkend="guc-restore-library"/> should return immediately if the file does not exist; the server will retry the command again if necessary. </para> @@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order. <para> If you're using a WAL archive, its size can be minimized using the <xref - linkend="guc-archive-cleanup-command"/> parameter to remove files that are no - longer required by the standby server. + linkend="guc-archive-cleanup-command"/> parameter or the + <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove + files that are no longer required by the standby server. The <application>pg_archivecleanup</application> utility is designed specifically to be used with <varname>archive_cleanup_command</varname> in typical single-standby configurations, see <xref linkend="pgarchivecleanup"/>. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e5d12f1d15..34a21c40a5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -83,7 +83,7 @@ #include "replication/snapbuild.h" #include "replication/walreceiver.h" #include "replication/walsender.h" -#include "restore/shell_restore.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog, TimeLineID newTLI) { /* - * Execute the recovery_end_command, if any. + * Execute the recovery-end callback, if any. * - * The command is provided the archive file cutoff point for use during + * The callback is provided the archive file cutoff point for use during * log shipping replication. All files earlier than this point can be * deleted from the archive, though there is no requirement to do so. */ - if (shell_recovery_end_configured()) + if (recovery_end_configured()) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_recovery_end(lastRestartPointFname); + RestoreCallbacks->recovery_end_cb(restore_module_state, + lastRestartPointFname); } /* @@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags) timestamptz_to_str(xtime)) : 0)); /* - * Finally, execute archive_cleanup_command, if any. + * Finally, execute archive-cleanup callback, if any. * - * The command is provided the archive file cutoff point for use during + * The callback is provided the archive file cutoff point for use during * log shipping replication. All files earlier than this point can be * deleted from the archive, though there is no requirement to do so. */ - if (shell_archive_cleanup_configured()) + if (archive_cleanup_configured()) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_archive_cleanup(lastRestartPointFname); + RestoreCallbacks->archive_cleanup_cb(restore_module_state, + lastRestartPointFname); } return true; diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 5756f52124..3a05581495 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -23,14 +23,21 @@ #include "access/xlog_internal.h" #include "access/xlogarchive.h" #include "common/archive.h" +#include "fmgr.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/pgarch.h" #include "postmaster/startup.h" #include "replication/walsender.h" +#include "restore/restore_module.h" #include "restore/shell_restore.h" #include "storage/fd.h" #include "storage/ipc.h" +#include "utils/memutils.h" + +char *restoreLibrary = ""; +const RestoreModuleCallbacks *RestoreCallbacks = NULL; +RestoreModuleState *restore_module_state; /* * Attempt to retrieve the specified file from off-line archival storage. @@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname, switch (archive_type) { case ARCHIVE_TYPE_WAL_SEGMENT: - if (!shell_restore_file_configured()) + if (!restore_wal_segment_configured()) goto not_available; break; case ARCHIVE_TYPE_TIMELINE_HISTORY: - if (!shell_restore_file_configured()) + if (!restore_timeline_history_configured()) goto not_available; break; case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: - if (!shell_restore_file_configured()) + if (!timeline_history_exists_configured()) goto not_available; break; } @@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname, switch (archive_type) { case ARCHIVE_TYPE_WAL_SEGMENT: - ret = shell_restore_wal_segment(xlogfname, xlogpath, - lastRestartPointFname); + ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state, + xlogfname, xlogpath, + lastRestartPointFname); break; case ARCHIVE_TYPE_TIMELINE_HISTORY: - ret = shell_restore_timeline_history(xlogfname, xlogpath); + ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state, + xlogfname, xlogpath); break; case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: - ret = shell_restore_timeline_history(xlogfname, xlogpath); + ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state, + xlogfname, xlogpath); break; } @@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname, if (ret) { + /* + * Some restore functions might not copy the file (e.g., checking + * whether a timeline history file exists), but they can set a flag to + * tell us if they do. We only need to verify file existence if this + * flag is enabled. + */ + if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS && + !restore_module_state->timeline_history_exists_cb_copies) + return true; + /* * command apparently succeeded, but let's make sure the file is * really there now and has the correct size. @@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog) unlink(archiveStatusPath); /* should we complain about failure? */ } + +/* + * Loads all the restore callbacks into our global RestoreCallbacks. The + * caller is responsible for validating the combination of library/command + * parameters that are set (e.g., restore_command and restore_library cannot + * both be set). + */ +void +LoadRestoreCallbacks(void) +{ + RestoreModuleInit init; + + /* + * If the shell command is enabled, use our special initialization + * function. Otherwise, load the library and call its + * _PG_restore_module_init(). + */ + if (restoreLibrary[0] == '\0') + init = shell_restore_init; + else + init = (RestoreModuleInit) + load_external_function(restoreLibrary, "_PG_restore_module_init", + false, NULL); + + if (init == NULL) + ereport(ERROR, + (errmsg("restore modules have to define the symbol " + "_PG_restore_module_init"))); + + RestoreCallbacks = (*init) (); + + /* restore state should be freed before calling this function */ + Assert(restore_module_state == NULL); + restore_module_state = (RestoreModuleState *) + MemoryContextAllocZero(TopMemoryContext, + sizeof(RestoreModuleState)); + + if (RestoreCallbacks->startup_cb != NULL) + RestoreCallbacks->startup_cb(restore_module_state); +} + +/* + * Call the shutdown callback of the loaded restore module, if defined. Also, + * free the restore module state if it was allocated. + * + * Processes that load restore libraries should register this via + * before_shmem_exit(). + */ +void +call_restore_module_shutdown_cb(int code, Datum arg) +{ + if (RestoreCallbacks != NULL && + RestoreCallbacks->shutdown_cb != NULL && + restore_module_state != NULL) + RestoreCallbacks->shutdown_cb(restore_module_state); + + if (restore_module_state != NULL) + { + pfree(restore_module_state); + restore_module_state = NULL; + } +} diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 07213b1897..1bf0d0ea11 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -51,6 +51,7 @@ #include "replication/slot.h" #include "replication/slotsync.h" #include "replication/walreceiver.h" +#include "restore/restore_module.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1117,18 +1118,21 @@ validateRecoveryParameters(void) if (StandbyModeRequested) { if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) && - (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)) + (!restore_wal_segment_configured() || + !restore_timeline_history_configured() || + !timeline_history_exists_configured())) ereport(WARNING, - (errmsg("specified neither primary_conninfo nor restore_command"), + (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"), errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there."))); } else { - if (recoveryRestoreCommand == NULL || - strcmp(recoveryRestoreCommand, "") == 0) + if (!restore_wal_segment_configured() || + !restore_timeline_history_configured() || + !timeline_history_exists_configured()) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("must specify restore_command when standby mode is not enabled"))); + errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled"))); } /* diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 8ef600ae72..c31d0f3bd2 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -46,6 +46,7 @@ #include "postmaster/bgwriter.h" #include "postmaster/interrupt.h" #include "replication/syncrep.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/condition_variable.h" #include "storage/fd.h" @@ -230,6 +231,25 @@ CheckpointerMain(char *startup_data, size_t startup_data_len) ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(checkpointer_context); + /* + * Load restore_library so that we can use its archive-cleanup callback, + * if one is defined. + * + * We also take this opportunity to set up the before_shmem_exit hook for + * the shutdown callback and to check that only one of restore_library, + * archive_cleanup_command is set. Note that we emit a WARNING upon + * detecting misconfigured parameters, and we clear the callbacks so that + * the archive-cleanup functionality is skipped. We judge this + * functionality to not be important enough to require blocking + * checkpoints or shutting down the server when the parameters are + * misconfigured. + */ + before_shmem_exit(call_restore_module_shutdown_cb, 0); + if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command", + WARNING)) + LoadRestoreCallbacks(); + /* * If an exception is encountered, processing resumes here. * @@ -562,6 +582,10 @@ HandleCheckpointerInterrupts(void) if (ConfigReloadPending) { + bool archive_cleanup_settings_changed = false; + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand); + ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); @@ -577,6 +601,36 @@ HandleCheckpointerInterrupts(void) * because of SIGHUP. */ UpdateSharedMemoryConfig(); + + /* + * If the archive-cleanup settings have changed, call the currently + * loaded shutdown callback and clear all the restore callbacks. + * There's presently no good way to unload a library besides + * restarting the process, and there should be little harm in leaving + * it around, so we just leave it loaded. + */ + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0) + { + archive_cleanup_settings_changed = true; + call_restore_module_shutdown_cb(0, (Datum) 0); + RestoreCallbacks = NULL; + } + + /* + * As in CheckpointerMain(), we only emit a WARNING if we detect that + * both restore_library and archive_cleanup_command are set. We do + * this even if the archive-cleanup settings haven't changed to remind + * the user about the misconfiguration. + */ + if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command", + WARNING) && + archive_cleanup_settings_changed) + LoadRestoreCallbacks(); + + pfree(prevRestoreLibrary); + pfree(prevArchiveCleanupCommand); } if (ShutdownRequestPending) { diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index cc665ce259..f8d3d6b3d2 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -26,6 +26,7 @@ #include "miscadmin.h" #include "postmaster/auxprocess.h" #include "postmaster/startup.h" +#include "restore/restore_module.h" #include "storage/ipc.h" #include "storage/pmsignal.h" #include "storage/procsignal.h" @@ -119,13 +120,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS) * Re-read the config file. * * If one of the critical walreceiver options has changed, flag xlog.c - * to restart it. + * to restart it. Also, check for invalid combinations of the command/library + * parameters and reload the restore callbacks if necessary. */ static void StartupRereadConfig(void) { char *conninfo = pstrdup(PrimaryConnInfo); char *slotname = pstrdup(PrimarySlotName); + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevRestoreCommand = pstrdup(recoveryRestoreCommand); + char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand); bool tempSlot = wal_receiver_create_temp_slot; bool conninfoChanged; bool slotnameChanged; @@ -147,6 +152,30 @@ StartupRereadConfig(void) if (conninfoChanged || slotnameChanged || tempSlotChanged) StartupRequestWalReceiverRestart(); + + /* + * If the restore settings have changed, call the currently loaded + * shutdown callback and load the new callbacks. There's presently no + * good way to unload a library besides restarting the process, and there + * should be little harm in leaving it around, so we just leave it loaded. + */ + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command", + ERROR); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command", + ERROR); + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 || + strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0) + { + call_restore_module_shutdown_cb(0, (Datum) 0); + LoadRestoreCallbacks(); + } + + pfree(prevRestoreLibrary); + pfree(prevRestoreCommand); + pfree(prevRecoveryEndCommand); } /* Handle various signals that might be sent to the startup process */ @@ -261,6 +290,19 @@ StartupProcessMain(char *startup_data, size_t startup_data_len) */ sigprocmask(SIG_SETMASK, &UnBlockSig, NULL); + /* + * Check for invalid combinations of the command/library parameters and + * load the callbacks. + */ + before_shmem_exit(call_restore_module_shutdown_cb, 0); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command", + ERROR); + (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command", + ERROR); + LoadRestoreCallbacks(); + /* * Do what we came for. */ diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c index 42291625c1..be25f04044 100644 --- a/src/backend/restore/shell_restore.c +++ b/src/backend/restore/shell_restore.c @@ -3,7 +3,8 @@ * shell_restore.c * * These restore functions use a user-specified shell command (e.g., the - * restore_command GUC). + * restore_command GUC). It is used as the default, but other modules may + * define their own restore logic. * * Copyright (c) 2024, PostgreSQL Global Development Group * @@ -22,25 +23,76 @@ #include "common/archive.h" #include "common/percentrepl.h" #include "postmaster/startup.h" +#include "restore/restore_module.h" #include "restore/shell_restore.h" #include "storage/ipc.h" #include "utils/wait_event.h" +static void shell_restore_startup(RestoreModuleState *state); +static bool shell_restore_file_configured(RestoreModuleState *state); static bool shell_restore_file(const char *file, const char *path, const char *lastRestartPointFileName); +static bool shell_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +static bool shell_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path); +static bool shell_archive_cleanup_configured(RestoreModuleState *state); +static void shell_archive_cleanup(RestoreModuleState *state, + const char *lastRestartPointFileName); +static bool shell_recovery_end_configured(RestoreModuleState *state); +static void shell_recovery_end(RestoreModuleState *state, + const char *lastRestartPointFileName); static void ExecuteRecoveryCommand(const char *command, const char *commandName, bool failOnSignal, uint32 wait_event_info, const char *lastRestartPointFileName); +static const RestoreModuleCallbacks shell_restore_callbacks = { + .startup_cb = shell_restore_startup, + .restore_wal_segment_configured_cb = shell_restore_file_configured, + .restore_wal_segment_cb = shell_restore_wal_segment, + .restore_timeline_history_configured_cb = shell_restore_file_configured, + .restore_timeline_history_cb = shell_restore_timeline_history, + .timeline_history_exists_configured_cb = shell_restore_file_configured, + .timeline_history_exists_cb = shell_restore_timeline_history, + .archive_cleanup_configured_cb = shell_archive_cleanup_configured, + .archive_cleanup_cb = shell_archive_cleanup, + .recovery_end_configured_cb = shell_recovery_end_configured, + .recovery_end_cb = shell_recovery_end, + .shutdown_cb = NULL +}; + +/* + * shell_restore_init + * + * Returns the callbacks for restoring via shell. + */ +const RestoreModuleCallbacks * +shell_restore_init(void) +{ + return &shell_restore_callbacks; +} + +/* + * Indicates that our timeline_history_exists_cb only attempts to retrieve the + * file, so the caller should verify the file exists afterwards. + */ +static void +shell_restore_startup(RestoreModuleState *state) +{ + state->timeline_history_exists_cb_copies = true; +} + /* * Attempt to execute a shell-based restore command to retrieve a WAL segment. * * Returns true if the command has succeeded, false otherwise. */ -bool -shell_restore_wal_segment(const char *file, const char *path, +static bool +shell_restore_wal_segment(RestoreModuleState *state, + const char *file, const char *path, const char *lastRestartPointFileName) { return shell_restore_file(file, path, lastRestartPointFileName); @@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path, * * Returns true if the command has succeeded, false otherwise. */ -bool -shell_restore_timeline_history(const char *file, const char *path) +static bool +shell_restore_timeline_history(RestoreModuleState *state, + const char *file, const char *path) { char lastRestartPointFname[MAXPGPATH]; @@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path) /* * Check whether restore_command is supplied. */ -bool -shell_restore_file_configured(void) +static bool +shell_restore_file_configured(RestoreModuleState *state) { return recoveryRestoreCommand[0] != '\0'; } @@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path, /* * Check whether archive_cleanup_command is supplied. */ -bool -shell_archive_cleanup_configured(void) +static bool +shell_archive_cleanup_configured(RestoreModuleState *state) { return archiveCleanupCommand[0] != '\0'; } @@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void) /* * Attempt to execute a shell-based archive cleanup command. */ -void -shell_archive_cleanup(const char *lastRestartPointFileName) +static void +shell_archive_cleanup(RestoreModuleState *state, + const char *lastRestartPointFileName) { ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command", false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND, @@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName) /* * Check whether recovery_end_command is supplied. */ -bool -shell_recovery_end_configured(void) +static bool +shell_recovery_end_configured(RestoreModuleState *state) { return recoveryEndCommand[0] != '\0'; } @@ -181,8 +235,9 @@ shell_recovery_end_configured(void) /* * Attempt to execute a shell-based end-of-recovery command. */ -void -shell_recovery_end(const char *lastRestartPointFileName) +static void +shell_recovery_end(RestoreModuleState *state, + const char *lastRestartPointFileName) { ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true, WAIT_EVENT_RECOVERY_END_COMMAND, diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1e71e7db4a..f84724a0ed 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -70,6 +70,7 @@ #include "replication/slot.h" #include "replication/slotsync.h" #include "replication/syncrep.h" +#include "restore/restore_module.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, + gettext_noop("Sets the library that will be called for restore actions."), + NULL + }, + &restoreLibrary, + "", + NULL, NULL, NULL + }, + { {"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, gettext_noop("Sets the shell command that will be executed at every restart point."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 2244ee52f7..5265857519 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -284,6 +284,7 @@ # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' +#restore_library = '' # library to use for restore actions #archive_cleanup_command = '' # command to execute at every restartpoint #recovery_end_command = '' # command to execute at completion of recovery diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h new file mode 100644 index 0000000000..cc148e855a --- /dev/null +++ b/src/include/restore/restore_module.h @@ -0,0 +1,143 @@ +/*------------------------------------------------------------------------- + * + * restore_module.h + * Exports for restore modules. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * src/include/restore/restore_module.h + * + *------------------------------------------------------------------------- + */ +#ifndef _RESTORE_MODULE_H +#define _RESTORE_MODULE_H + +/* + * The value of the restore_library GUC. + */ +extern PGDLLIMPORT char *restoreLibrary; + +typedef struct RestoreModuleState +{ + /* + * Private data pointer for use by a restore module. This can be used to + * store state for the module that will be passed to each of its + * callbacks. + */ + void *private_data; + + /* + * Indicates whether the callback that checks for timeline existence + * merely copies the file. If set to true, the server will verify that + * the timeline history file exists after the callback returns. + */ + bool timeline_history_exists_cb_copies; +} RestoreModuleState; + +/* + * Restore module callbacks + * + * These callback functions should be defined by restore libraries and returned + * via _PG_restore_module_init(). For more information about the purpose of + * each callback, refer to the restore modules documentation. + */ +typedef void (*RestoreStartupCB) (RestoreModuleState *state); +typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state); +typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, + const char *file, const char *path, + const char *lastRestartPointFileName); +typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state); +typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, + const char *file, const char *path); +typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state); +typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, + const char *file, const char *path); +typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state); +typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, + const char *lastRestartPointFileName); +typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state); +typedef void (*RecoveryEndCB) (RestoreModuleState *state, + const char *lastRestartPointFileName); +typedef void (*RestoreShutdownCB) (RestoreModuleState *state); + +typedef struct RestoreModuleCallbacks +{ + RestoreStartupCB startup_cb; + RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb; + RestoreWalSegmentCB restore_wal_segment_cb; + RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb; + RestoreTimelineHistoryCB restore_timeline_history_cb; + TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb; + TimelineHistoryExistsCB timeline_history_exists_cb; + ArchiveCleanupConfiguredCB archive_cleanup_configured_cb; + ArchiveCleanupCB archive_cleanup_cb; + RecoveryEndConfiguredCB recovery_end_configured_cb; + RecoveryEndCB recovery_end_cb; + RestoreShutdownCB shutdown_cb; +} RestoreModuleCallbacks; + +/* + * Type of the shared library symbol _PG_restore_module_init that is looked up + * when loading a restore library. + */ +typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void); + +extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void); + +extern void LoadRestoreCallbacks(void); +extern void call_restore_module_shutdown_cb(int code, Datum arg); + +extern const RestoreModuleCallbacks *RestoreCallbacks; +extern RestoreModuleState *restore_module_state; + +/* + * The following *_configured() functions are convenient wrappers around the + * *_configured_cb() callback functions with additional defensive NULL checks. + */ + +static inline bool +restore_wal_segment_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->restore_wal_segment_configured_cb != NULL && + RestoreCallbacks->restore_wal_segment_cb != NULL && + RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state); +} + +static inline bool +restore_timeline_history_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->restore_timeline_history_configured_cb != NULL && + RestoreCallbacks->restore_timeline_history_cb != NULL && + RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state); +} + +static inline bool +timeline_history_exists_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->timeline_history_exists_configured_cb != NULL && + RestoreCallbacks->timeline_history_exists_cb != NULL && + RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state); +} + +static inline bool +archive_cleanup_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->archive_cleanup_configured_cb != NULL && + RestoreCallbacks->archive_cleanup_cb != NULL && + RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state); +} + +static inline bool +recovery_end_configured(void) +{ + return RestoreCallbacks != NULL && + RestoreCallbacks->recovery_end_configured_cb != NULL && + RestoreCallbacks->recovery_end_cb != NULL && + RestoreCallbacks->recovery_end_configured_cb(restore_module_state); +} + +#endif /* _RESTORE_MODULE_H */ diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h index 28b5b7bc92..6352623866 100644 --- a/src/include/restore/shell_restore.h +++ b/src/include/restore/shell_restore.h @@ -12,15 +12,13 @@ #ifndef _SHELL_RESTORE_H #define _SHELL_RESTORE_H -extern bool shell_restore_file_configured(void); -extern bool shell_restore_wal_segment(const char *file, const char *path, - const char *lastRestartPointFileName); -extern bool shell_restore_timeline_history(const char *file, const char *path); +#include "restore/restore_module.h" -extern bool shell_archive_cleanup_configured(void); -extern void shell_archive_cleanup(const char *lastRestartPointFileName); - -extern bool shell_recovery_end_configured(void); -extern void shell_recovery_end(const char *lastRestartPointFileName); +/* + * Since the logic for restoring via shell commands is in the core server and + * does not need to be loaded via a shared library, it has a special + * initialization function. + */ +extern const RestoreModuleCallbacks *shell_restore_init(void); #endif /* _SHELL_RESTORE_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a40c59b2e0..db4cc68a88 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2405,6 +2405,8 @@ ResourceOwnerDesc ResourceReleaseCallback ResourceReleaseCallbackItem ResourceReleasePhase +RestoreModuleCallbacks +RestoreModuleState RestoreOptions RestorePass RestrictInfo -- 2.25.1 --3V7upXqbjpZ4EhLz-- ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-15 06:27 Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-15 06:27 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, If there are no more comments for the current design, I'll start implementing this feature with the following approaches for "Discussing" items: > 3.1 Should we use one function(internal) for COPY TO/FROM > handlers or two function(internal)s (one is for COPY TO > handler and another is for COPY FROM handler)? > [4] I'll choose "one function(internal) for COPY TO/FROM handlers". > 3.4 Should we export Copy{To,From}State? Or should we just > provide getters/setters to access Copy{To,From}State > internal? > [10] I'll export Copy{To,From}State. Thanks, -- kou In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > Hi, > > Here is the current summary for a this discussion to make > COPY format extendable. It's for reaching consensus and > starting implementing the feature. (I'll start implementing > the feature once we reach consensus.) If you have any > opinion, please share it. > > Confirmed: > > 1.1 Making COPY format extendable will not reduce performance. > [1] > > Decisions: > > 2.1 Use separated handler for COPY TO and COPY FROM because > our COPY TO implementation (copyto.c) and COPY FROM > implementation (coypfrom.c) are separated. > [2] > > 2.2 Don't use system catalog for COPY TO/FROM handlers. We can > just use a function(internal) that returns a handler instead. > [3] > > 2.3 The implementation must include documentation. > [5] > > 2.4 The implementation must include test. > [6] > > 2.5 The implementation should be consist of small patches > for easy to review. > [6] > > 2.7 Copy{To,From}State must have a opaque space for > handlers. > [8] > > 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO > handlers. > [8] > > 2.9 Make "format" in PgMsg_CopyOutResponse message > extendable. > [9] > > 2.10 Make makeNode() call avoidable in function(internal) > that returns COPY TO/FROM handler. > [9] > > 2.11 Custom COPY TO/FROM handlers must be able to parse > their options. > [11] > > Discussing: > > 3.1 Should we use one function(internal) for COPY TO/FROM > handlers or two function(internal)s (one is for COPY TO > handler and another is for COPY FROM handler)? > [4] > > 3.2 If we use separated function(internal) for COPY TO/FROM > handlers, we need to define naming rule. For example, > <method_name>_to(internal) for COPY TO handler and > <method_name>_from(internal) for COPY FROM handler. > [4] > > 3.3 Should we use prefix or suffix for function(internal) > name to avoid name conflict with other handlers such as > tablesample handlers? > [7] > > 3.4 Should we export Copy{To,From}State? Or should we just > provide getters/setters to access Copy{To,From}State > internal? > [10] > > > [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com > [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz > [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m... > [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m... > [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp... > [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz > [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com > [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com > [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz > [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m... > [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com > > > Thanks, > -- > kou > > ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 05:49 Sutou Kouhei <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 2 replies; 38+ messages in thread From: Sutou Kouhei @ 2024-01-24 05:49 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, I've implemented custom COPY format feature based on the current design discussion. See the attached patches for details. I also implemented a PoC COPY format handler for Apache Arrow with this implementation and it worked. https://github.com/kou/pg-copy-arrow The patches implement not only custom COPY TO format feature but also custom COPY FROM format feature. 0001-0004 is for COPY TO and 0005-0008 is for COPY FROM. For COPY TO: 0001: This adds CopyToRoutine and use it for text/csv/binary formats. No implementation change. This just move codes. 0002: This adds support for adding custom COPY TO format by "CREATE FUNCTION ${FORMAT_NAME}". This uses the same approach provided by Sawada-san[1] but this doesn't introduce a wrapper CopyRoutine struct for Copy{To,From}Routine. Because I noticed that a wrapper CopyRoutine struct is needless. Copy handler can just return CopyToRoutine or CopyFromRtouine because both of them have NodeTag. We can distinct a returned struct by the NodeTag. [1] https://www.postgresql.org/message-id/[email protected]... 0003: This exports CopyToStateData. No implementation change except CopyDest enum values. I changed COPY_ prefix to COPY_DEST_ to avoid name conflict with CopySource enum values. This just moves codes. 0004: This adds CopyToState::opaque and exports CopySendEndOfRow(). CopySendEndOfRow() is renamed to CopyToStateFlush(). For COPY FROM: 0005: Same as 0001 but for COPY FROM. This adds CopyFromRoutine and use it for text/csv/binary formats. No implementation change. This just move codes. 0006: Same as 0002 but for COPY FROM. This adds support for adding custom COPY FROM format by "CREATE FUNCTION ${FORMAT_NAME}". 0007: Same as 0003 but for COPY FROM. This exports CopyFromStateData. No implementation change except CopySource enum values. I changed COPY_ prefix to COPY_SOURCE_ to align CopyDest changes in 0003. This just moves codes. 0008: Same as 0004 but for COPY FROM. This adds CopyFromState::opaque and exports CopyReadBinaryData(). CopyReadBinaryData() is renamed to CopyFromStateRead(). Thanks, -- kou In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Jan 2024 15:27:02 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > Hi, > > If there are no more comments for the current design, I'll > start implementing this feature with the following > approaches for "Discussing" items: > >> 3.1 Should we use one function(internal) for COPY TO/FROM >> handlers or two function(internal)s (one is for COPY TO >> handler and another is for COPY FROM handler)? >> [4] > > I'll choose "one function(internal) for COPY TO/FROM handlers". > >> 3.4 Should we export Copy{To,From}State? Or should we just >> provide getters/setters to access Copy{To,From}State >> internal? >> [10] > > I'll export Copy{To,From}State. > > > Thanks, > -- > kou > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST), > Sutou Kouhei <[email protected]> wrote: > >> Hi, >> >> Here is the current summary for a this discussion to make >> COPY format extendable. It's for reaching consensus and >> starting implementing the feature. (I'll start implementing >> the feature once we reach consensus.) If you have any >> opinion, please share it. >> >> Confirmed: >> >> 1.1 Making COPY format extendable will not reduce performance. >> [1] >> >> Decisions: >> >> 2.1 Use separated handler for COPY TO and COPY FROM because >> our COPY TO implementation (copyto.c) and COPY FROM >> implementation (coypfrom.c) are separated. >> [2] >> >> 2.2 Don't use system catalog for COPY TO/FROM handlers. We can >> just use a function(internal) that returns a handler instead. >> [3] >> >> 2.3 The implementation must include documentation. >> [5] >> >> 2.4 The implementation must include test. >> [6] >> >> 2.5 The implementation should be consist of small patches >> for easy to review. >> [6] >> >> 2.7 Copy{To,From}State must have a opaque space for >> handlers. >> [8] >> >> 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO >> handlers. >> [8] >> >> 2.9 Make "format" in PgMsg_CopyOutResponse message >> extendable. >> [9] >> >> 2.10 Make makeNode() call avoidable in function(internal) >> that returns COPY TO/FROM handler. >> [9] >> >> 2.11 Custom COPY TO/FROM handlers must be able to parse >> their options. >> [11] >> >> Discussing: >> >> 3.1 Should we use one function(internal) for COPY TO/FROM >> handlers or two function(internal)s (one is for COPY TO >> handler and another is for COPY FROM handler)? >> [4] >> >> 3.2 If we use separated function(internal) for COPY TO/FROM >> handlers, we need to define naming rule. For example, >> <method_name>_to(internal) for COPY TO handler and >> <method_name>_from(internal) for COPY FROM handler. >> [4] >> >> 3.3 Should we use prefix or suffix for function(internal) >> name to avoid name conflict with other handlers such as >> tablesample handlers? >> [7] >> >> 3.4 Should we export Copy{To,From}State? Or should we just >> provide getters/setters to access Copy{To,From}State >> internal? >> [10] >> >> >> [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com >> [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz >> [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m... >> [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m... >> [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp... >> [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz >> [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com >> [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com >> [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz >> [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m... >> [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com >> >> >> Thanks, >> -- >> kou >> >> > > ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 08:11 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-24 08:11 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: > For COPY TO: > > 0001: This adds CopyToRoutine and use it for text/csv/binary > formats. No implementation change. This just move codes. 10M without this change: format,elapsed time (ms) text,1090.763 csv,1136.103 binary,1137.141 10M with this change: format,elapsed time (ms) text,1082.654 csv,1196.991 binary,1069.697 These numbers point out that binary is faster by 6%, csv is slower by 5%, while text stays around what looks like noise range. That's not negligible. Are these numbers reproducible? If they are, that could be a problem for anybody doing bulk-loading of large data sets. I am not sure to understand where the improvement for binary comes from by reading the patch, but perhaps perf would tell more for each format? The loss with csv could be blamed on the extra manipulations of the function pointers, likely. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 12:15 Andrew Dunstan <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Andrew Dunstan @ 2024-01-24 12:15 UTC (permalink / raw) To: Michael Paquier <[email protected]>; Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers On 2024-01-24 We 03:11, Michael Paquier wrote: > On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: >> For COPY TO: >> >> 0001: This adds CopyToRoutine and use it for text/csv/binary >> formats. No implementation change. This just move codes. > 10M without this change: > > format,elapsed time (ms) > text,1090.763 > csv,1136.103 > binary,1137.141 > > 10M with this change: > > format,elapsed time (ms) > text,1082.654 > csv,1196.991 > binary,1069.697 > > These numbers point out that binary is faster by 6%, csv is slower by > 5%, while text stays around what looks like noise range. That's not > negligible. Are these numbers reproducible? If they are, that could > be a problem for anybody doing bulk-loading of large data sets. I am > not sure to understand where the improvement for binary comes from by > reading the patch, but perhaps perf would tell more for each format? > The loss with csv could be blamed on the extra manipulations of the > function pointers, likely. I don't think that's at all acceptable. We've spent quite a lot of blood sweat and tears over the years to make COPY fast, and we should not sacrifice any of that lightly. cheers andrew -- Andrew Dunstan EDB: https://www.enterprisedb.com ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 14:17 Sutou Kouhei <[email protected]> parent: Andrew Dunstan <[email protected]> 0 siblings, 3 replies; 38+ messages in thread From: Sutou Kouhei @ 2024-01-24 14:17 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, Andrew Dunstan <[email protected]> wrote: > > On 2024-01-24 We 03:11, Michael Paquier wrote: >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: >>> For COPY TO: >>> >>> 0001: This adds CopyToRoutine and use it for text/csv/binary >>> formats. No implementation change. This just move codes. >> 10M without this change: >> >> format,elapsed time (ms) >> text,1090.763 >> csv,1136.103 >> binary,1137.141 >> >> 10M with this change: >> >> format,elapsed time (ms) >> text,1082.654 >> csv,1196.991 >> binary,1069.697 >> >> These numbers point out that binary is faster by 6%, csv is slower by >> 5%, while text stays around what looks like noise range. That's not >> negligible. Are these numbers reproducible? If they are, that could >> be a problem for anybody doing bulk-loading of large data sets. I am >> not sure to understand where the improvement for binary comes from by >> reading the patch, but perhaps perf would tell more for each format? >> The loss with csv could be blamed on the extra manipulations of the >> function pointers, likely. > > > I don't think that's at all acceptable. > > We've spent quite a lot of blood sweat and tears over the years to make COPY > fast, and we should not sacrifice any of that lightly. These numbers aren't reproducible. Because these benchmarks executed on my normal machine not a machine only for benchmarking. The machine runs another processes such as editor and Web browser. For example, here are some results with master (94edfe250c6a200d2067b0debfe00b4122e9b11e): Format,N records,Elapsed time (ms) csv,10000000,1073.715 csv,10000000,1022.830 csv,10000000,1073.584 csv,10000000,1090.651 csv,10000000,1052.259 Here are some results with master + the 0001 patch: Format,N records,Elapsed time (ms) csv,10000000,1025.356 csv,10000000,1067.202 csv,10000000,1014.563 csv,10000000,1032.088 csv,10000000,1058.110 I uploaded my benchmark script so that you can run the same benchmark on your machine: https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 Could anyone try the benchmark with master and master+0001? Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-24 14:20 Sutou Kouhei <[email protected]> parent: Sutou Kouhei <[email protected]> 1 sibling, 0 replies; 38+ messages in thread From: Sutou Kouhei @ 2024-01-24 14:20 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 14:49:36 +0900 (JST), Sutou Kouhei <[email protected]> wrote: > I've implemented custom COPY format feature based on the > current design discussion. See the attached patches for > details. I forgot to mention one note. Documentation isn't included in these patches. I'll write it after all (or some) patches are merged. Is it OK? Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 02:53 jian he <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 2 replies; 38+ messages in thread From: jian he @ 2024-01-25 02:53 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 10:17 PM Sutou Kouhei <[email protected]> wrote: > > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 > > Could anyone try the benchmark with master and master+0001? > sorry. I made a mistake. I applied v6, 0001 to 0008 all the patches. my tests: CREATE unlogged TABLE data (a bigint); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, 1e7); my setup: meson setup --reconfigure ${BUILD} \ -Dprefix=${PG_PREFIX} \ -Dpgport=5462 \ -Dbuildtype=release \ -Ddocs_html_style=website \ -Ddocs_pdf=disabled \ -Dllvm=disabled \ -Dextra_version=_release_build gcc version: PostgreSQL 17devel_release_build on x86_64-linux, compiled by gcc-11.4.0, 64-bit apply your patch: COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Time: 668.996 ms Time: 596.254 ms Time: 592.723 ms Time: 591.663 ms Time: 590.803 ms not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Time: 644.246 ms Time: 583.075 ms Time: 568.670 ms Time: 569.463 ms Time: 569.201 ms I forgot to test other formats. ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 03:17 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-25 03:17 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 11:17:26PM +0900, Sutou Kouhei wrote: > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, > Andrew Dunstan <[email protected]> wrote: >> We've spent quite a lot of blood sweat and tears over the years to make COPY >> fast, and we should not sacrifice any of that lightly. Clearly. > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 Thanks, that saves time. I am attaching it to this email as well, for the sake of the archives if this link is removed in the future. > Could anyone try the benchmark with master and master+0001? Yep. It is one point we need to settle before deciding what to do with this patch set, and I've done so to reach my own conclusion. I have a rather good machine at my disposal in the cloud, so I did a few runs with HEAD and HEAD+0001, with PGDATA mounted on a tmpfs. Here are some results for the 10M row case, as these should be the least prone to noise, 5 runs each: master text 10M 1732.570 1684.542 1693.430 1687.696 1714.845 csv 10M 1729.113 1724.926 1727.414 1726.237 1728.865 bin 10M 1679.097 1677.887 1676.764 1677.554 1678.120 master+0001 text 10M 1702.207 1654.818 1647.069 1690.568 1654.446 csv 10M 1764.939 1714.313 1712.444 1712.323 1716.952 bin 10M 1703.061 1702.719 1702.234 1703.346 1704.137 Hmm. The point of contention in the patch is the change to use the CopyToOneRow callback in CopyOneRowTo(), as we go through it for each row and we should habe a worst-case scenario with a relation that has a small attribute size. The more rows, the more effect it would have. The memory context switches and the StringInfo manipulations are equally important, and there are a bunch of the latter, actually, with optimizations around fe_msgbuf. I've repeated a few runs across these two builds, and there is some variance and noise, but I am going to agree with your point that the effect 0001 cannot be seen. Even HEAD is showing some noise. So I am discarding the concerns I had after seeing the numbers you posted upthread. +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); +typedef void (*CopyToEnd_function) (CopyToState cstate); We don't really need a set of typedefs here, let's put the definitions in the CopyToRoutine struct instead. +extern CopyToRoutine CopyToRoutineText; +extern CopyToRoutine CopyToRoutineCSV; +extern CopyToRoutine CopyToRoutineBinary; All that should IMO remain in copyto.c and copyfrom.c in the initial patch doing the refactoring. Why not using a fetch function instead that uses a string in input? Then you can call that once after parsing the List of options in ProcessCopyOptions(). Introducing copyapi.h in the initial patch makes sense here for the TO and FROM routines. +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ Some areas, like this comment, are written in an incorrect format. + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, colname); You are right that this is not worth the trouble of creating a different set of callbacks for CSV. This makes the result cleaner. + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); Actually, this split is interesting. It is possible for a custom format to plug in a custom set of out functions. Did you make use of something custom for your own stuff? Actually, could it make sense to split the assignment of cstate->out_functions into its own callback? Sure, that's part of the start phase, but at least it would make clear that a custom method *has* to assign these OIDs to work. The patch implies that as a rule, without a comment that CopyToStart *must* set up these OIDs. I think that 0001 and 0005 should be handled first, as pieces independent of the rest. Then we could move on with 0002~0004 and 0006~0008. -- Michael #!/bin/bash set -eu set -o pipefail base_dir=$(dirname "$0") prepare_sql() { size=$1 db_name=$2 cat <<SQL DROP DATABASE IF EXISTS ${db_name}; CREATE DATABASE ${db_name}; \\c ${db_name} CREATE TABLE data (int32 integer); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${size}); SQL } measure() { local format=$1 local n_tries=6 for i in $(seq ${n_tries}); do LANG=C \ PAGER=cat \ psql \ --no-psqlrc \ --command "\\timing" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \ grep --text -o '^Time: [0-9.]* ms' | \ grep -o '[0-9.]*' done | sort --numeric-sort | head -n 9 | tail -n 1 } result_suffix="" if [ $# -gt 0 ]; then result_suffix="-$1" fi result="${base_dir}/result${result_suffix}.csv" echo "Format,N records,Elapsed time (ms)" | \ tee "${result}" sizes=() sizes+=(100000) sizes+=(1000000) sizes+=(10000000) for size in "${sizes[@]}"; do export PGDATABASE="copy_${size}" echo "${size}: preparing" prepare_sql "${size}" "${PGDATABASE}" | \ psql -d postgres echo "${size}: text" elapsed_time=$(measure text) echo "text,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: csv" elapsed_time=$(measure csv) echo "csv,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: binary" elapsed_time=$(measure binary) echo "binary,${size},${elapsed_time}" | \ tee -a "${result}" done Attachments: [text/plain] bench-run.txt (1.6K, ../../[email protected]/2-bench-run.txt) download | inline: #!/bin/bash set -eu set -o pipefail base_dir=$(dirname "$0") prepare_sql() { size=$1 db_name=$2 cat <<SQL DROP DATABASE IF EXISTS ${db_name}; CREATE DATABASE ${db_name}; \\c ${db_name} CREATE TABLE data (int32 integer); SELECT setseed(0.29); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${size}); SQL } measure() { local format=$1 local n_tries=6 for i in $(seq ${n_tries}); do LANG=C \ PAGER=cat \ psql \ --no-psqlrc \ --command "\\timing" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \ --command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \ grep --text -o '^Time: [0-9.]* ms' | \ grep -o '[0-9.]*' done | sort --numeric-sort | head -n 9 | tail -n 1 } result_suffix="" if [ $# -gt 0 ]; then result_suffix="-$1" fi result="${base_dir}/result${result_suffix}.csv" echo "Format,N records,Elapsed time (ms)" | \ tee "${result}" sizes=() sizes+=(100000) sizes+=(1000000) sizes+=(10000000) for size in "${sizes[@]}"; do export PGDATABASE="copy_${size}" echo "${size}: preparing" prepare_sql "${size}" "${PGDATABASE}" | \ psql -d postgres echo "${size}: text" elapsed_time=$(measure text) echo "text,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: csv" elapsed_time=$(measure csv) echo "csv,${size},${elapsed_time}" | \ tee -a "${result}" echo "${size}: binary" elapsed_time=$(measure binary) echo "binary,${size},${elapsed_time}" | \ tee -a "${result}" done [application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 03:28 Michael Paquier <[email protected]> parent: jian he <[email protected]> 1 sibling, 0 replies; 38+ messages in thread From: Michael Paquier @ 2024-01-25 03:28 UTC (permalink / raw) To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 10:53:58AM +0800, jian he wrote: > apply your patch: > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 > Time: 668.996 ms > Time: 596.254 ms > Time: 592.723 ms > Time: 591.663 ms > Time: 590.803 ms > > not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 > Time: 644.246 ms > Time: 583.075 ms > Time: 568.670 ms > Time: 569.463 ms > Time: 569.201 ms > > I forgot to test other formats. There can be some variance in the tests, so you'd better run much more tests so as you can get a better idea of the mean. Discarding the N highest and lowest values also reduces slightly the effects of the noise you would get across single runs. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 04:36 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 2 siblings, 2 replies; 38+ messages in thread From: Masahiko Sawada @ 2024-01-25 04:36 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 24, 2024 at 11:17 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500, > Andrew Dunstan <[email protected]> wrote: > > > > > On 2024-01-24 We 03:11, Michael Paquier wrote: > >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote: > >>> For COPY TO: > >>> > >>> 0001: This adds CopyToRoutine and use it for text/csv/binary > >>> formats. No implementation change. This just move codes. > >> 10M without this change: > >> > >> format,elapsed time (ms) > >> text,1090.763 > >> csv,1136.103 > >> binary,1137.141 > >> > >> 10M with this change: > >> > >> format,elapsed time (ms) > >> text,1082.654 > >> csv,1196.991 > >> binary,1069.697 > >> > >> These numbers point out that binary is faster by 6%, csv is slower by > >> 5%, while text stays around what looks like noise range. That's not > >> negligible. Are these numbers reproducible? If they are, that could > >> be a problem for anybody doing bulk-loading of large data sets. I am > >> not sure to understand where the improvement for binary comes from by > >> reading the patch, but perhaps perf would tell more for each format? > >> The loss with csv could be blamed on the extra manipulations of the > >> function pointers, likely. > > > > > > I don't think that's at all acceptable. > > > > We've spent quite a lot of blood sweat and tears over the years to make COPY > > fast, and we should not sacrifice any of that lightly. > > These numbers aren't reproducible. Because these benchmarks > executed on my normal machine not a machine only for > benchmarking. The machine runs another processes such as > editor and Web browser. > > For example, here are some results with master > (94edfe250c6a200d2067b0debfe00b4122e9b11e): > > Format,N records,Elapsed time (ms) > csv,10000000,1073.715 > csv,10000000,1022.830 > csv,10000000,1073.584 > csv,10000000,1090.651 > csv,10000000,1052.259 > > Here are some results with master + the 0001 patch: > > Format,N records,Elapsed time (ms) > csv,10000000,1025.356 > csv,10000000,1067.202 > csv,10000000,1014.563 > csv,10000000,1032.088 > csv,10000000,1058.110 > > > I uploaded my benchmark script so that you can run the same > benchmark on your machine: > > https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5 > > Could anyone try the benchmark with master and master+0001? > I've run a similar scenario: create unlogged table test (a int); insert into test select c from generate_series(1, 25000000) c; copy test to '/tmp/result.csv' with (format csv); -- generates 230MB file I've run it on HEAD and HEAD+0001 patch and here are the medians of 10 executions for each format: HEAD: binary 2930.353 ms text 2754.852 ms csv 2890.012 ms HEAD w/ 0001 patch: binary 2814.838 ms text 2900.845 ms csv 3015.210 ms Hmm I can see a similar trend that Suto-san had; the binary format got slightly faster whereas both text and csv format has small regression (4%~5%). I think that the improvement for binary came from the fact that we removed "if (cstate->opts.binary)" branches from the original CopyOneRowTo(). I've experimented with a similar optimization for csv and text format; have different callbacks for text and csv format and remove "if (cstate->opts.csv_mode)" branches. I've attached a patch for that. Here are results: HEAD w/ 0001 patch + remove branches: binary 2824.502 ms text 2715.264 ms csv 2803.381 ms The numbers look better now. I'm not sure these are within a noise range but it might be worth considering having different callbacks for text and csv formats. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com Attachments: [application/octet-stream] add_callback_for_csv_format.patch (3.9K, ../../CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com/2-add_callback_for_csv_format.patch) download | inline diff: diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6547b7c654..f18b7d0823 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -223,11 +223,7 @@ CopyToTextStart(CopyToState cstate, TupleDesc tupDesc) colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, colname, false, - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, colname); + CopyAttributeOutText(cstate, colname); } CopyToTextSendEndOfRow(cstate); @@ -260,12 +256,7 @@ CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot) char *string; string = OutputFunctionCall(&out_functions[attnum - 1], value); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, string, - cstate->opts.force_quote_flags[attnum - 1], - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, string); + CopyAttributeOutText(cstate, string); } } @@ -277,6 +268,99 @@ CopyToTextEnd(CopyToState cstate) { } +static void +CopyToCSVStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + /* + * For non-binary copy, we need to convert null_print to file encoding, + * because it will be sent directly with CopySendString. + */ + if (cstate->need_transcoding) + cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, + cstate->opts.null_print_len, + cstate->file_encoding); + + /* if a header has been requested send the line */ + if (cstate->opts.header_line) + { + bool hdr_delim = false; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + char *colname; + + if (hdr_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + hdr_delim = true; + + colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); + + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + } + + CopyToTextSendEndOfRow(cstate); + } +} + +static void +CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + bool need_delim = false; + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (need_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + need_delim = true; + + if (isnull) + { + CopySendString(cstate, cstate->opts.null_print_client); + } + else + { + char *string; + + string = OutputFunctionCall(&out_functions[attnum - 1], value); + CopyAttributeOutCSV(cstate, string, + cstate->opts.force_quote_flags[attnum - 1], + list_length(cstate->attnumlist) == 1); + } + } + + CopyToTextSendEndOfRow(cstate); +} + +static void +CopyToCSVEnd(CopyToState cstate) +{ +} + /* * CopyToRoutine implementation for "binary". */ @@ -388,9 +472,9 @@ CopyToRoutine CopyToRoutineText = { CopyToRoutine CopyToRoutineCSV = { .CopyToProcessOption = CopyToTextProcessOption, .CopyToGetFormat = CopyToTextGetFormat, - .CopyToStart = CopyToTextStart, - .CopyToOneRow = CopyToTextOneRow, - .CopyToEnd = CopyToTextEnd, + .CopyToStart = CopyToCSVStart, + .CopyToOneRow = CopyToCSVOneRow, + .CopyToEnd = CopyToCSVEnd, }; CopyToRoutine CopyToRoutineBinary = { ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 04:53 Michael Paquier <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-25 04:53 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote: > Hmm I can see a similar trend that Suto-san had; the binary format got > slightly faster whereas both text and csv format has small regression > (4%~5%). I think that the improvement for binary came from the fact > that we removed "if (cstate->opts.binary)" branches from the original > CopyOneRowTo(). I've experimented with a similar optimization for csv > and text format; have different callbacks for text and csv format and > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > for that. Here are results: > > HEAD w/ 0001 patch + remove branches: > binary 2824.502 ms > text 2715.264 ms > csv 2803.381 ms > > The numbers look better now. I'm not sure these are within a noise > range but it might be worth considering having different callbacks for > text and csv formats. Interesting. Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and 0.9% speedup for binary, which may be around the noise range assuming a ~1% range. While this does not imply a regression, that seems worth the duplication IMO. The patch had better document the reason why the split is done, as well. CopyFromTextOneRow() has also specific branches for binary and non-binary removed in 0005, so assuming that I/O is not a bottleneck, the operation would be faster because we would not evaluate this "if" condition for each row. Wouldn't we also see improvements for COPY FROM with short row values, say when mounting PGDATA into a tmpfs/ramfs? -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 05:28 Masahiko Sawada <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 0 replies; 38+ messages in thread From: Masahiko Sawada @ 2024-01-25 05:28 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 1:53 PM Michael Paquier <[email protected]> wrote: > > On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote: > > Hmm I can see a similar trend that Suto-san had; the binary format got > > slightly faster whereas both text and csv format has small regression > > (4%~5%). I think that the improvement for binary came from the fact > > that we removed "if (cstate->opts.binary)" branches from the original > > CopyOneRowTo(). I've experimented with a similar optimization for csv > > and text format; have different callbacks for text and csv format and > > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > > for that. Here are results: > > > > HEAD w/ 0001 patch + remove branches: > > binary 2824.502 ms > > text 2715.264 ms > > csv 2803.381 ms > > > > The numbers look better now. I'm not sure these are within a noise > > range but it might be worth considering having different callbacks for > > text and csv formats. > > Interesting. > > Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and > 0.9% speedup for binary, which may be around the noise range assuming > a ~1% range. While this does not imply a regression, that seems worth > the duplication IMO. Agreed. In addition to that, now that each format routine has its own callbacks, there would be chances that we can do other optimizations dedicated to the format type in the future if available. > The patch had better document the reason why the > split is done, as well. +1 > > CopyFromTextOneRow() has also specific branches for binary and > non-binary removed in 0005, so assuming that I/O is not a bottleneck, > the operation would be faster because we would not evaluate this "if" > condition for each row. Wouldn't we also see improvements for COPY > FROM with short row values, say when mounting PGDATA into a > tmpfs/ramfs? Probably. Seems worth evaluating. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:05 Sutou Kouhei <[email protected]> parent: jian he <[email protected]> 1 sibling, 0 replies; 38+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:05 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, Thanks for trying these patches! In <CACJufxF9NS3xQ2d79jN0V1CGvF7cR16uJo-C3nrY7vZrwvxF7w@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 10:53:58 +0800, jian he <[email protected]> wrote: > COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5 Wow! I didn't know the "\watch count="! I'll use it. > Time: 668.996 ms > Time: 596.254 ms > Time: 592.723 ms > Time: 591.663 ms > Time: 590.803 ms It seems that 5 times isn't enough for this case as Michael said. But thanks for trying! Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:45 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900, Michael Paquier <[email protected]> wrote: > +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); > +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); > +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); > +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); > +typedef void (*CopyToEnd_function) (CopyToState cstate); > > We don't really need a set of typedefs here, let's put the definitions > in the CopyToRoutine struct instead. OK. I'll do it. > +extern CopyToRoutine CopyToRoutineText; > +extern CopyToRoutine CopyToRoutineCSV; > +extern CopyToRoutine CopyToRoutineBinary; > > All that should IMO remain in copyto.c and copyfrom.c in the initial > patch doing the refactoring. Why not using a fetch function instead > that uses a string in input? Then you can call that once after > parsing the List of options in ProcessCopyOptions(). OK. How about the following for the fetch function signature? extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); We may introduce an enum and use it: typedef enum CopyBuiltinFormat { COPY_BUILTIN_FORMAT_TEXT = 0, COPY_BUILTIN_FORMAT_CSV, COPY_BUILTIN_FORMAT_BINARY, } CopyBuiltinFormat; extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format); > +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may > + * move the code to here later. */ > Some areas, like this comment, are written in an incorrect format. Oh, sorry. I assumed that the comment style was adjusted by pgindent. I'll use the following style: /* * ... */ > + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); > + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); > > Actually, this split is interesting. It is possible for a custom > format to plug in a custom set of out functions. Did you make use of > something custom for your own stuff? I didn't. My PoC custom COPY format handler for Apache Arrow just handles integer and text for now. It doesn't use cstate->out_functions because cstate->out_functions may not return a valid binary format value for Apache Arrow. So it formats each value by itself. I'll chose one of them for a custom type (that isn't supported by Apache Arrow, e.g. PostGIS types): 1. Report an unsupported error 2. Call output function for Apache Arrow provided by the custom type > Actually, could it make sense to > split the assignment of cstate->out_functions into its own callback? Yes. Because we need to use getTypeBinaryOutputInfo() for "binary" and use getTypeOutputInfo() for "text" and "csv". > Sure, that's part of the start phase, but at least it would make clear > that a custom method *has* to assign these OIDs to work. The patch > implies that as a rule, without a comment that CopyToStart *must* set > up these OIDs. CopyToStart doesn't need to set up them if the handler doesn't use cstate->out_functions. > I think that 0001 and 0005 should be handled first, as pieces > independent of the rest. Then we could move on with 0002~0004 and > 0006~0008. OK. I'll focus on 0001 and 0005 for now. I'll restart 0002-0004/0006-0008 after 0001 and 0005 are accepted. Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 08:52 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-25 08:52 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900, Masahiko Sawada <[email protected]> wrote: > I've experimented with a similar optimization for csv > and text format; have different callbacks for text and csv format and > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > for that. Here are results: > > HEAD w/ 0001 patch + remove branches: > binary 2824.502 ms > text 2715.264 ms > csv 2803.381 ms > > The numbers look better now. I'm not sure these are within a noise > range but it might be worth considering having different callbacks for > text and csv formats. Wow! Interesting. I tried the approach before but I didn't see any difference by the approach. But it may depend on my environment. I'll import the approach to the next patch set so that others can try the approach easily. Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-25 23:35 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-25 23:35 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 05:45:43PM +0900, Sutou Kouhei wrote: > In <[email protected]> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900, > Michael Paquier <[email protected]> wrote: >> +extern CopyToRoutine CopyToRoutineText; >> +extern CopyToRoutine CopyToRoutineCSV; >> +extern CopyToRoutine CopyToRoutineBinary; >> >> All that should IMO remain in copyto.c and copyfrom.c in the initial >> patch doing the refactoring. Why not using a fetch function instead >> that uses a string in input? Then you can call that once after >> parsing the List of options in ProcessCopyOptions(). > > OK. How about the following for the fetch function > signature? > > extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad history with naming things around here. > We may introduce an enum and use it: > > typedef enum CopyBuiltinFormat > { > COPY_BUILTIN_FORMAT_TEXT = 0, > COPY_BUILTIN_FORMAT_CSV, > COPY_BUILTIN_FORMAT_BINARY, > } CopyBuiltinFormat; > > extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format); I am not sure that this is necessary as the option value is a string. > Oh, sorry. I assumed that the comment style was adjusted by > pgindent. No worries, that's just something we get used to. I tend to fix a lot of these things by myself when editing patches. >> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); >> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); >> >> Actually, this split is interesting. It is possible for a custom >> format to plug in a custom set of out functions. Did you make use of >> something custom for your own stuff? > > I didn't. My PoC custom COPY format handler for Apache Arrow > just handles integer and text for now. It doesn't use > cstate->out_functions because cstate->out_functions may not > return a valid binary format value for Apache Arrow. So it > formats each value by itself. I mean, if you use a custom output function, you could tweak things even more with byteas or such.. If a callback is expected to do something, like setting the output function OIDs in the start callback, we'd better document it rather than letting that be implied. >> Actually, could it make sense to >> split the assignment of cstate->out_functions into its own callback? > > Yes. Because we need to use getTypeBinaryOutputInfo() for > "binary" and use getTypeOutputInfo() for "text" and "csv". Okay. After sleeping on it, a split makes sense here, because it also reduces the presence of TupleDesc in the start callback. >> Sure, that's part of the start phase, but at least it would make clear >> that a custom method *has* to assign these OIDs to work. The patch >> implies that as a rule, without a comment that CopyToStart *must* set >> up these OIDs. > > CopyToStart doesn't need to set up them if the handler > doesn't use cstate->out_functions. Noted. >> I think that 0001 and 0005 should be handled first, as pieces >> independent of the rest. Then we could move on with 0002~0004 and >> 0006~0008. > > OK. I'll focus on 0001 and 0005 for now. I'll restart > 0002-0004/0006-0008 after 0001 and 0005 are accepted. Once you get these, I'd be interested in re-doing an evaluation of COPY TO and more tests with COPY FROM while running Postgres on scissors. One thing I was thinking to use here is my blackhole_am for COPY FROM: https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am As per its name, it does nothing on INSERT, so you could create a table using it as access method, and stress the COPY FROM execution paths without having to mount Postgres on a tmpfs because the data is sent to the void. Perhaps it does not matter, but that moves the tests to the bottlenecks we want to stress (aka the per-row callback for large data sets). I've switched the patch as waiting on author for now. Thanks for your perseverance here. I understand that's not easy to follow up with patches and reviews (^_^;) -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:18 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Junwang Zhao @ 2024-01-26 08:18 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Thu, Jan 25, 2024 at 4:52 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900, > Masahiko Sawada <[email protected]> wrote: > > > I've experimented with a similar optimization for csv > > and text format; have different callbacks for text and csv format and > > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch > > for that. Here are results: > > > > HEAD w/ 0001 patch + remove branches: > > binary 2824.502 ms > > text 2715.264 ms > > csv 2803.381 ms > > > > The numbers look better now. I'm not sure these are within a noise > > range but it might be worth considering having different callbacks for > > text and csv formats. > > Wow! Interesting. I tried the approach before but I didn't > see any difference by the approach. But it may depend on my > environment. > > I'll import the approach to the next patch set so that > others can try the approach easily. > > > Thanks, > -- > kou Hi Kou-san, In the current implementation, there is no way that one can check incompatibility options in ProcessCopyOptions, we can postpone the check in CopyFromStart or CopyToStart, but I think it is a little bit late. Do you think adding an extra check for incompatible options hook is acceptable (PFA)? -- Regards Junwang Zhao Attachments: [application/octet-stream] 0001-add-check-incomptiblity-options-hooks.patch (3.1K, ../../CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com/2-0001-add-check-incomptiblity-options-hooks.patch) download | inline diff: From f01ad61fe6c251a00870ace5d37669350f8e2043 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Fri, 26 Jan 2024 15:55:07 +0800 Subject: [PATCH] add check incomptiblity options hooks Signed-off-by: Zhao Junwang <[email protected]> --- src/backend/commands/copy.c | 5 +++++ src/include/commands/copyapi.h | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 479f36868c..985d50870f 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -924,6 +924,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("NULL specification and DEFAULT specification cannot be the same"))); } + + if (is_from) + opts_out->from_routine->CopyFromCheckIncompatibleOptions(cstate, opts_out); + else + opts_out->to_routine->CopyToCheckIncompatibleOptions(cstate, opts_out); } /* diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 22accc83ab..34aa86761a 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -21,8 +21,10 @@ #include "nodes/parsenodes.h" typedef struct CopyFromStateData *CopyFromState; +typedef struct CopyFormatOptions; typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); +typedef void (*CopyFromCheckIncompatibleOptions_function) (CopyFromState cstate, CopyFormatOptions *options); typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate); typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc); typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); @@ -39,6 +41,12 @@ typedef struct CopyFromRoutine */ CopyFromProcessOption_function CopyFromProcessOption; + /* + * Called for processing incompatible options. This will report error + * messages when find incompatible options. + */ + CopyFromCheckIncompatibleOptions_function CopyFromCheckIncompatibleOptions; + /* * Called when COPY FROM is started. This will return a format as int16 * value. It's used for the CopyInResponse message. @@ -67,6 +75,7 @@ extern CopyFromRoutine CopyFromRoutineBinary; typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef void (*CopyToCheckIncompatibleOptions_function) (CopyToState cstate, CopyFormatOptions *options); typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); @@ -83,6 +92,12 @@ typedef struct CopyToRoutine */ CopyToProcessOption_function CopyToProcessOption; + /* + * Called for processing incompatible options. This will report error + * messages when find incompatible options. + */ + CopyToCheckIncompatibleOptions_function CopyToCheckIncompatibleOptions; + /* * Called when COPY TO is started. This will return a format as int16 * value. It's used for the CopyOutResponse message. -- 2.41.0 ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:32 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:32 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800, Junwang Zhao <[email protected]> wrote: > In the current implementation, there is no way that one can check > incompatibility > options in ProcessCopyOptions, we can postpone the check in CopyFromStart > or CopyToStart, but I think it is a little bit late. Do you think > adding an extra > check for incompatible options hook is acceptable (PFA)? Thanks for the suggestion! But I think that a custom handler can do it in CopyToProcessOption()/CopyFromProcessOption(). What do you think about this? Or could you share a sample COPY TO/FROM WITH() SQL you think? Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:41 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Junwang Zhao @ 2024-01-26 08:41 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 4:32 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800, > Junwang Zhao <[email protected]> wrote: > > > In the current implementation, there is no way that one can check > > incompatibility > > options in ProcessCopyOptions, we can postpone the check in CopyFromStart > > or CopyToStart, but I think it is a little bit late. Do you think > > adding an extra > > check for incompatible options hook is acceptable (PFA)? > > Thanks for the suggestion! But I think that a custom handler > can do it in > CopyToProcessOption()/CopyFromProcessOption(). What do you > think about this? Or could you share a sample COPY TO/FROM > WITH() SQL you think? CopyToProcessOption()/CopyFromProcessOption() can only handle single option, and store the options in the opaque field, but it can not check the relation of two options, for example, considering json format, the `header` option can not be handled by these two functions. I want to find a way when the user specifies the header option, customer handler can error out. > > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:49 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 0 replies; 38+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:49 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 08:35:19 +0900, Michael Paquier <[email protected]> wrote: >> OK. How about the following for the fetch function >> signature? >> >> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format); > > Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad > history with naming things around here. Thanks for the suggestion. I rethink about this and use the following: +extern void ProcessCopyOptionFormatTo(ParseState *pstate, CopyFormatOptions *opts_out, DefElem *defel); It's not a fetch function. It sets CopyToRoutine opts_out instead. But it hides CopyToRoutine* to copyto.c. Is it acceptable? >> OK. I'll focus on 0001 and 0005 for now. I'll restart >> 0002-0004/0006-0008 after 0001 and 0005 are accepted. > > Once you get these, I'd be interested in re-doing an evaluation of > COPY TO and more tests with COPY FROM while running Postgres on > scissors. One thing I was thinking to use here is my blackhole_am for > COPY FROM: > https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am Thanks! Could you evaluate the attached patch set with COPY FROM? I attach v7 patch set. It includes only the 0001 and 0005 parts in v6 patch set because we focus on them for now. 0001: This is based on 0001 in v6. Changes since v6: * Fix comment style * Hide CopyToRoutine{Text,CSV,Binary} * Add more comments * Eliminate "if (cstate->opts.csv_mode)" branches from "text" and "csv" callbacks * Remove CopyTo*_function typedefs * Update benchmark results in commit message but the results are measured on my environment that isn't suitable for accurate benchmark 0002: This is based on 0005 in v6. Changes since v6: * Fix comment style * Hide CopyFromRoutine{Text,CSV,Binary} * Add more comments * Eliminate a "if (cstate->opts.csv_mode)" branch from "text" and "csv" callbacks * NOTE: We can eliminate more "if (cstate->opts.csv_mode)" branches such as one in NextCopyFromRawFields(). Should we do it in this feature improvement (make COPY format extendable)? Can we defer this as a separated improvement? * Remove CopyFrom*_function typedefs Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 08:55 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-26 08:55 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, Junwang Zhao <[email protected]> wrote: > CopyToProcessOption()/CopyFromProcessOption() can only handle > single option, and store the options in the opaque field, but it can not > check the relation of two options, for example, considering json format, > the `header` option can not be handled by these two functions. > > I want to find a way when the user specifies the header option, customer > handler can error out. Ah, you want to use a built-in option (such as "header") value from a custom handler, right? Hmm, it may be better that we call CopyToProcessOption()/CopyFromProcessOption() for all options including built-in options. Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-26 09:02 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 2 replies; 38+ messages in thread From: Junwang Zhao @ 2024-01-26 09:02 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > Junwang Zhao <[email protected]> wrote: > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > single option, and store the options in the opaque field, but it can not > > check the relation of two options, for example, considering json format, > > the `header` option can not be handled by these two functions. > > > > I want to find a way when the user specifies the header option, customer > > handler can error out. > > Ah, you want to use a built-in option (such as "header") > value from a custom handler, right? Hmm, it may be better > that we call CopyToProcessOption()/CopyFromProcessOption() > for all options including built-in options. > Hmm, still I don't think it can handle all cases, since we don't know the sequence of the options, we need all the options been parsed before we check the compatibility of the options, or customer handlers will need complicated logic to resolve that, which might lead to ugly code :( > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-27 06:15 Junwang Zhao <[email protected]> parent: Junwang Zhao <[email protected]> 1 sibling, 1 reply; 38+ messages in thread From: Junwang Zhao @ 2024-01-27 06:15 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi Kou-san, On Fri, Jan 26, 2024 at 5:02 PM Junwang Zhao <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > Junwang Zhao <[email protected]> wrote: > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > single option, and store the options in the opaque field, but it can not > > > check the relation of two options, for example, considering json format, > > > the `header` option can not be handled by these two functions. > > > > > > I want to find a way when the user specifies the header option, customer > > > handler can error out. > > > > Ah, you want to use a built-in option (such as "header") > > value from a custom handler, right? Hmm, it may be better > > that we call CopyToProcessOption()/CopyFromProcessOption() > > for all options including built-in options. > > > Hmm, still I don't think it can handle all cases, since we don't know > the sequence of the options, we need all the options been parsed > before we check the compatibility of the options, or customer > handlers will need complicated logic to resolve that, which might > lead to ugly code :( > I have been working on a *COPY TO JSON* extension since yesterday, which is based on your V6 patch set, I'd like to give you more input so you can make better decisions about the implementation(with only pg-copy-arrow you might not get everything considered). V8 is based on V6, so anybody involved in the performance issue should still review the V7 patch set. 0001-0008 is your original V6 implementations 0009 is some changes made by me, I changed CopyToGetFormat to CopyToSendCopyBegin because pg_copy_json need to send different bytes in SendCopyBegin, get the format code along is not enough, I once had a thought that may be we should merge SendCopyBegin/SendCopyEnd into CopyToStart/CopyToEnd but I don't do that in this patch. I have also exported more APIs for extension usage. 00010 is the pg_copy_json extension, I think this should be a good case which can utilize the *extendable copy format* feature, maybe we should delete copy_test_format if we have this extension as an example? > > > > Thanks, > > -- > > kou > > > > -- > Regards > Junwang Zhao -- Regards Junwang Zhao Attachments: [application/octet-stream] v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch (3.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/2-v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch) download | inline diff: From 4b177469f3fb8f14f0cd6bff3c7878dcafd9b760 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 15:12:43 +0900 Subject: [PATCH v8 04/10] Add support for implementing custom COPY TO format as extension * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). --- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cfc74ee7b1..b5d8678394 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) default: break; } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } CopyToRoutine CopyToRoutineText = { @@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index a869d78d72..ffad433a21 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -174,6 +174,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0001-Extract-COPY-TO-format-implementations.patch (23.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/3-v8-0001-Extract-COPY-TO-format-implementations.patch) download | inline diff: From 6e68ba6380dc825a242e7f0d0a53442bba3a4a61 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Mon, 4 Dec 2023 12:32:54 +0900 Subject: [PATCH v8 01/10] Extract COPY TO format implementations This is a part of making COPY format extendable. See also these past discussions: * New Copy Formats - avro/orc/parquet: https://www.postgresql.org/message-id/flat/20180210151304.fonjztsynewldfba%40gmail.com * Make COPY extendable in order to support Parquet and other formats: https://www.postgresql.org/message-id/flat/CAJ7c6TM6Bz1c3F04Cy6%2BSzuWfKmr0kU8c_3Stnvh_8BR0D6k8Q%40mail.gmail.com This doesn't change the current behavior. This just introduces CopyToRoutine, which just has function pointers of format implementation like TupleTableSlotOps, and use it for existing "text", "csv" and "binary" format implementations. Note that CopyToRoutine can't be used from extensions yet because CopySend*() aren't exported yet. Extensions can't send formatted data to a destination without CopySend*(). They will be exported by subsequent patches. Here is a benchmark result with/without this change because there was a discussion that we should care about performance regression: https://www.postgresql.org/message-id/3741749.1655952719%40sss.pgh.pa.us > I think that step 1 ought to be to convert the existing formats into > plug-ins, and demonstrate that there's no significant loss of > performance. You can see that there is no significant loss of performance: Data: Random 32 bit integers: CREATE TABLE data (int32 integer); INSERT INTO data SELECT random() * 10000 FROM generate_series(1, ${n_records}); The number of records: 100K, 1M and 10M 100K without this change: format,elapsed time (ms) text,11.002 csv,11.696 binary,11.352 100K with this change: format,elapsed time (ms) text,100000,11.562 csv,100000,11.889 binary,100000,10.825 1M without this change: format,elapsed time (ms) text,108.359 csv,114.233 binary,111.251 1M with this change: format,elapsed time (ms) text,111.269 csv,116.277 binary,104.765 10M without this change: format,elapsed time (ms) text,1090.763 csv,1136.103 binary,1137.141 10M with this change: format,elapsed time (ms) text,1082.654 csv,1196.991 binary,1069.697 --- contrib/file_fdw/file_fdw.c | 2 +- src/backend/commands/copy.c | 43 +++- src/backend/commands/copyfrom.c | 2 +- src/backend/commands/copyto.c | 428 ++++++++++++++++++++------------ src/include/commands/copy.h | 7 +- src/include/commands/copyapi.h | 59 +++++ 6 files changed, 376 insertions(+), 165 deletions(-) create mode 100644 src/include/commands/copyapi.h diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 249d82d3a0..9e4e819858 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -329,7 +329,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Now apply the core COPY code's validation logic for more checks. */ - ProcessCopyOptions(NULL, NULL, true, other_options); + ProcessCopyOptions(NULL, NULL, true, NULL, other_options); /* * Either filename or program option is required for file_fdw foreign diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index cc0786c6f4..5f3697a5f9 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -442,6 +442,9 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from) * a list of options. In that usage, 'opts_out' can be passed as NULL and * the collected data is just leaked until CurrentMemoryContext is reset. * + * 'cstate' is CopyToState* for !is_from, CopyFromState* for is_from. 'cstate' + * may be NULL. For example, file_fdw uses NULL. + * * Note that additional checking, such as whether column names listed in FORCE * QUOTE actually exist, has to be applied later. This just checks for * self-consistency of the options list. @@ -450,6 +453,7 @@ void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, + void *cstate, List *options) { bool format_specified = false; @@ -464,7 +468,13 @@ ProcessCopyOptions(ParseState *pstate, opts_out->file_encoding = -1; - /* Extract options from the statement node tree */ + /* Text is the default format. */ + opts_out->to_routine = &CopyToRoutineText; + + /* + * Extract only the "format" option to detect target routine as the first + * step + */ foreach(option, options) { DefElem *defel = lfirst_node(DefElem, option); @@ -479,15 +489,29 @@ ProcessCopyOptions(ParseState *pstate, if (strcmp(fmt, "text") == 0) /* default format */ ; else if (strcmp(fmt, "csv") == 0) + { opts_out->csv_mode = true; + opts_out->to_routine = &CopyToRoutineCSV; + } else if (strcmp(fmt, "binary") == 0) + { opts_out->binary = true; + opts_out->to_routine = &CopyToRoutineBinary; + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY format \"%s\" not recognized", fmt), parser_errposition(pstate, defel->location))); } + } + /* Extract options except "format" from the statement node tree */ + foreach(option, options) + { + DefElem *defel = lfirst_node(DefElem, option); + + if (strcmp(defel->defname, "format") == 0) + continue; else if (strcmp(defel->defname, "freeze") == 0) { if (freeze_specified) @@ -616,11 +640,18 @@ ProcessCopyOptions(ParseState *pstate, opts_out->on_error = defGetCopyOnErrorChoice(defel, pstate, is_from); } else - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", - defel->defname), - parser_errposition(pstate, defel->location))); + { + bool processed = false; + + if (!is_from) + processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); + if (!processed) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("option \"%s\" not recognized", + defel->defname), + parser_errposition(pstate, defel->location))); + } } /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 1fe70b9133..fb3d4d9296 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1416,7 +1416,7 @@ BeginCopyFrom(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options); + ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , cstate, options); /* Process the target relation */ cstate->rel = rel; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index d3dc3fc854..6547b7c654 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -131,6 +131,275 @@ static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); +/* + * CopyToRoutine implementations. + */ + +/* + * CopyToRoutine implementation for "text" and "csv". CopyToText*() + * refer cstate->opts.csv_mode and change their behavior. We can split this + * implementation and stop referring cstate->opts.csv_mode later. + */ + +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ +static bool +CopyToTextProcessOption(CopyToState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyToTextGetFormat(CopyToState cstate) +{ + return 0; +} + +static void +CopyToTextSendEndOfRow(CopyToState cstate) +{ + switch (cstate->copy_dest) + { + case COPY_FILE: + /* Default line termination depends on platform */ +#ifndef WIN32 + CopySendChar(cstate, '\n'); +#else + CopySendString(cstate, "\r\n"); +#endif + break; + case COPY_FRONTEND: + /* The FE/BE protocol uses \n as newline for all platforms */ + CopySendChar(cstate, '\n'); + break; + default: + break; + } + CopySendEndOfRow(cstate); +} + +static void +CopyToTextStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + /* + * For non-binary copy, we need to convert null_print to file encoding, + * because it will be sent directly with CopySendString. + */ + if (cstate->need_transcoding) + cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, + cstate->opts.null_print_len, + cstate->file_encoding); + + /* if a header has been requested send the line */ + if (cstate->opts.header_line) + { + bool hdr_delim = false; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + char *colname; + + if (hdr_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + hdr_delim = true; + + colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); + + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, colname, false, + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, colname); + } + + CopyToTextSendEndOfRow(cstate); + } +} + +static void +CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + bool need_delim = false; + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (need_delim) + CopySendChar(cstate, cstate->opts.delim[0]); + need_delim = true; + + if (isnull) + { + CopySendString(cstate, cstate->opts.null_print_client); + } + else + { + char *string; + + string = OutputFunctionCall(&out_functions[attnum - 1], value); + if (cstate->opts.csv_mode) + CopyAttributeOutCSV(cstate, string, + cstate->opts.force_quote_flags[attnum - 1], + list_length(cstate->attnumlist) == 1); + else + CopyAttributeOutText(cstate, string); + } + } + + CopyToTextSendEndOfRow(cstate); +} + +static void +CopyToTextEnd(CopyToState cstate) +{ +} + +/* + * CopyToRoutine implementation for "binary". + */ + +/* All "binary" options are parsed in ProcessCopyOptions(). We may move the + * code to here later. */ +static bool +CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyToBinaryGetFormat(CopyToState cstate) +{ + return 1; +} + +static void +CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc) +{ + int num_phys_attrs; + ListCell *cur; + + num_phys_attrs = tupDesc->natts; + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } + + { + /* Generate header for a binary copy */ + int32 tmp; + + /* Signature */ + CopySendData(cstate, BinarySignature, 11); + /* Flags field */ + tmp = 0; + CopySendInt32(cstate, tmp); + /* No header extension */ + tmp = 0; + CopySendInt32(cstate, tmp); + } +} + +static void +CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + FmgrInfo *out_functions = cstate->out_functions; + ListCell *cur; + + /* Binary per-tuple header */ + CopySendInt16(cstate, list_length(cstate->attnumlist)); + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Datum value = slot->tts_values[attnum - 1]; + bool isnull = slot->tts_isnull[attnum - 1]; + + if (isnull) + { + CopySendInt32(cstate, -1); + } + else + { + bytea *outputbytes; + + outputbytes = SendFunctionCall(&out_functions[attnum - 1], value); + CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ); + CopySendData(cstate, VARDATA(outputbytes), + VARSIZE(outputbytes) - VARHDRSZ); + } + } + + CopySendEndOfRow(cstate); +} + +static void +CopyToBinaryEnd(CopyToState cstate) +{ + /* Generate trailer for a binary copy */ + CopySendInt16(cstate, -1); + /* Need to flush out the trailer */ + CopySendEndOfRow(cstate); +} + +CopyToRoutine CopyToRoutineText = { + .CopyToProcessOption = CopyToTextProcessOption, + .CopyToGetFormat = CopyToTextGetFormat, + .CopyToStart = CopyToTextStart, + .CopyToOneRow = CopyToTextOneRow, + .CopyToEnd = CopyToTextEnd, +}; + +/* + * We can use the same CopyToRoutine for both of "text" and "csv" because + * CopyToText*() refer cstate->opts.csv_mode and change their behavior. We can + * split the implementations and stop referring cstate->opts.csv_mode later. + */ +CopyToRoutine CopyToRoutineCSV = { + .CopyToProcessOption = CopyToTextProcessOption, + .CopyToGetFormat = CopyToTextGetFormat, + .CopyToStart = CopyToTextStart, + .CopyToOneRow = CopyToTextOneRow, + .CopyToEnd = CopyToTextEnd, +}; + +CopyToRoutine CopyToRoutineBinary = { + .CopyToProcessOption = CopyToBinaryProcessOption, + .CopyToGetFormat = CopyToBinaryGetFormat, + .CopyToStart = CopyToBinaryStart, + .CopyToOneRow = CopyToBinaryOneRow, + .CopyToEnd = CopyToBinaryEnd, +}; /* * Send copy start/stop messages for frontend copies. These have changed @@ -141,7 +410,7 @@ SendCopyBegin(CopyToState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate); int i; pq_beginmessage(&buf, PqMsg_CopyOutResponse); @@ -198,16 +467,6 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { case COPY_FILE: - if (!cstate->opts.binary) - { - /* Default line termination depends on platform */ -#ifndef WIN32 - CopySendChar(cstate, '\n'); -#else - CopySendString(cstate, "\r\n"); -#endif - } - if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || ferror(cstate->copy_file)) @@ -242,10 +501,6 @@ CopySendEndOfRow(CopyToState cstate) } break; case COPY_FRONTEND: - /* The FE/BE protocol uses \n as newline for all platforms */ - if (!cstate->opts.binary) - CopySendChar(cstate, '\n'); - /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; @@ -431,7 +686,7 @@ BeginCopyTo(ParseState *pstate, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options); + ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , cstate, options); /* Process the source/target relation or query */ if (rel) @@ -748,8 +1003,6 @@ DoCopyTo(CopyToState cstate) bool pipe = (cstate->filename == NULL && cstate->data_dest_cb == NULL); bool fe_copy = (pipe && whereToSendOutput == DestRemote); TupleDesc tupDesc; - int num_phys_attrs; - ListCell *cur; uint64 processed; if (fe_copy) @@ -759,32 +1012,11 @@ DoCopyTo(CopyToState cstate) tupDesc = RelationGetDescr(cstate->rel); else tupDesc = cstate->queryDesc->tupDesc; - num_phys_attrs = tupDesc->natts; cstate->opts.null_print_client = cstate->opts.null_print; /* default */ /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ cstate->fe_msgbuf = makeStringInfo(); - /* Get info about the columns we need to process. */ - cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - Oid out_func_oid; - bool isvarlena; - Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - - if (cstate->opts.binary) - getTypeBinaryOutputInfo(attr->atttypid, - &out_func_oid, - &isvarlena); - else - getTypeOutputInfo(attr->atttypid, - &out_func_oid, - &isvarlena); - fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); - } - /* * Create a temporary memory context that we can reset once per row to * recover palloc'd memory. This avoids any problems with leaks inside @@ -795,57 +1027,7 @@ DoCopyTo(CopyToState cstate) "COPY TO", ALLOCSET_DEFAULT_SIZES); - if (cstate->opts.binary) - { - /* Generate header for a binary copy */ - int32 tmp; - - /* Signature */ - CopySendData(cstate, BinarySignature, 11); - /* Flags field */ - tmp = 0; - CopySendInt32(cstate, tmp); - /* No header extension */ - tmp = 0; - CopySendInt32(cstate, tmp); - } - else - { - /* - * For non-binary copy, we need to convert null_print to file - * encoding, because it will be sent directly with CopySendString. - */ - if (cstate->need_transcoding) - cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print, - cstate->opts.null_print_len, - cstate->file_encoding); - - /* if a header has been requested send the line */ - if (cstate->opts.header_line) - { - bool hdr_delim = false; - - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - char *colname; - - if (hdr_delim) - CopySendChar(cstate, cstate->opts.delim[0]); - hdr_delim = true; - - colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, colname, false, - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, colname); - } - - CopySendEndOfRow(cstate); - } - } + cstate->opts.to_routine->CopyToStart(cstate, tupDesc); if (cstate->rel) { @@ -884,13 +1066,7 @@ DoCopyTo(CopyToState cstate) processed = ((DR_copy *) cstate->queryDesc->dest)->processed; } - if (cstate->opts.binary) - { - /* Generate trailer for a binary copy */ - CopySendInt16(cstate, -1); - /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); - } + cstate->opts.to_routine->CopyToEnd(cstate); MemoryContextDelete(cstate->rowcontext); @@ -906,71 +1082,15 @@ DoCopyTo(CopyToState cstate) static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) { - bool need_delim = false; - FmgrInfo *out_functions = cstate->out_functions; MemoryContext oldcontext; - ListCell *cur; - char *string; MemoryContextReset(cstate->rowcontext); oldcontext = MemoryContextSwitchTo(cstate->rowcontext); - if (cstate->opts.binary) - { - /* Binary per-tuple header */ - CopySendInt16(cstate, list_length(cstate->attnumlist)); - } - /* Make sure the tuple is fully deconstructed */ slot_getallattrs(slot); - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - Datum value = slot->tts_values[attnum - 1]; - bool isnull = slot->tts_isnull[attnum - 1]; - - if (!cstate->opts.binary) - { - if (need_delim) - CopySendChar(cstate, cstate->opts.delim[0]); - need_delim = true; - } - - if (isnull) - { - if (!cstate->opts.binary) - CopySendString(cstate, cstate->opts.null_print_client); - else - CopySendInt32(cstate, -1); - } - else - { - if (!cstate->opts.binary) - { - string = OutputFunctionCall(&out_functions[attnum - 1], - value); - if (cstate->opts.csv_mode) - CopyAttributeOutCSV(cstate, string, - cstate->opts.force_quote_flags[attnum - 1], - list_length(cstate->attnumlist) == 1); - else - CopyAttributeOutText(cstate, string); - } - else - { - bytea *outputbytes; - - outputbytes = SendFunctionCall(&out_functions[attnum - 1], - value); - CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ); - CopySendData(cstate, VARDATA(outputbytes), - VARSIZE(outputbytes) - VARHDRSZ); - } - } - } - - CopySendEndOfRow(cstate); + cstate->opts.to_routine->CopyToOneRow(cstate, slot); MemoryContextSwitchTo(oldcontext); } diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index b3da3cb0be..34bea880ca 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -14,6 +14,7 @@ #ifndef COPY_H #define COPY_H +#include "commands/copyapi.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -74,11 +75,11 @@ typedef struct CopyFormatOptions bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ List *convert_select; /* list of column names (can be NIL) */ + CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; -/* These are private in commands/copy[from|to].c */ +/* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; -typedef struct CopyToStateData *CopyToState; typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); typedef void (*copy_data_dest_cb) (void *data, int len); @@ -87,7 +88,7 @@ extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); -extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options); +extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options); extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *whereClause, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h new file mode 100644 index 0000000000..eb68f2fb7b --- /dev/null +++ b/src/include/commands/copyapi.h @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------- + * + * copyapi.h + * API for COPY TO/FROM handlers + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/commands/copyapi.h + * + *------------------------------------------------------------------------- + */ +#ifndef COPYAPI_H +#define COPYAPI_H + +#include "executor/tuptable.h" +#include "nodes/parsenodes.h" + +/* This is private in commands/copyto.c */ +typedef struct CopyToStateData *CopyToState; + +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); +typedef void (*CopyToEnd_function) (CopyToState cstate); + +/* Routines for a COPY TO format implementation. */ +typedef struct CopyToRoutine +{ + /* + * Called for processing one COPY TO option. This will return false when + * the given option is invalid. + */ + CopyToProcessOption_function CopyToProcessOption; + + /* + * Called when COPY TO is started. This will return a format as int16 + * value. It's used for the CopyOutResponse message. + */ + CopyToGetFormat_function CopyToGetFormat; + + /* Called when COPY TO is started. This will send a header. */ + CopyToStart_function CopyToStart; + + /* Copy one row for COPY TO. */ + CopyToOneRow_function CopyToOneRow; + + /* Called when COPY TO is ended. This will send a trailer. */ + CopyToEnd_function CopyToEnd; +} CopyToRoutine; + +/* Built-in CopyToRoutine for "text", "csv" and "binary". */ +extern CopyToRoutine CopyToRoutineText; +extern CopyToRoutine CopyToRoutineCSV; +extern CopyToRoutine CopyToRoutineBinary; + +#endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch (17.7K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/4-v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch) download | inline diff: From a597f8a2beec12971d77419f08b5722f531774f3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 13:58:38 +0900 Subject: [PATCH v8 02/10] Add support for adding custom COPY TO format This uses the handler approach like tablesample. The approach creates an internal function that returns an internal struct. In this case, a COPY TO handler returns a CopyToRoutine. We will add support for custom COPY FROM format later. We'll use the same handler for COPY TO and COPY FROM. PostgreSQL calls a COPY TO/FROM handler with "is_from" argument. It's true for COPY FROM and false for COPY TO: copy_handler(true) returns CopyToRoutine copy_handler(false) returns CopyFromRoutine (not exist yet) We discussed that we introduce a wrapper struct for it: typedef struct CopyRoutine { NodeTag type; /* either CopyToRoutine or CopyFromRoutine */ Node *routine; } copy_handler(true) returns CopyRoutine with CopyToRoutine copy_handler(false) returns CopyRoutine with CopyFromRoutine See also: https://www.postgresql.org/message-id/flat/CAD21AoCunywHird3GaPzWe6s9JG1wzxj3Cr6vGN36DDheGjOjA%40mail.gmail.com But I noticed that we don't need the wrapper struct. We can just CopyToRoutine or CopyFromRoutine. Because we can distinct the returned struct by checking its NodeTag. So I don't use the wrapper struct approach. --- src/backend/commands/copy.c | 84 ++++++++++++++----- src/backend/nodes/Makefile | 1 + src/backend/nodes/gen_node_support.pl | 2 + src/backend/utils/adt/pseudotypes.c | 1 + src/include/catalog/pg_proc.dat | 6 ++ src/include/catalog/pg_type.dat | 6 ++ src/include/commands/copyapi.h | 2 + src/include/nodes/meson.build | 1 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_copy_format/.gitignore | 4 + src/test/modules/test_copy_format/Makefile | 23 +++++ .../expected/test_copy_format.out | 17 ++++ src/test/modules/test_copy_format/meson.build | 33 ++++++++ .../test_copy_format/sql/test_copy_format.sql | 8 ++ .../test_copy_format--1.0.sql | 8 ++ .../test_copy_format/test_copy_format.c | 77 +++++++++++++++++ .../test_copy_format/test_copy_format.control | 4 + 18 files changed, 260 insertions(+), 19 deletions(-) mode change 100644 => 100755 src/backend/nodes/gen_node_support.pl create mode 100644 src/test/modules/test_copy_format/.gitignore create mode 100644 src/test/modules/test_copy_format/Makefile create mode 100644 src/test/modules/test_copy_format/expected/test_copy_format.out create mode 100644 src/test/modules/test_copy_format/meson.build create mode 100644 src/test/modules/test_copy_format/sql/test_copy_format.sql create mode 100644 src/test/modules/test_copy_format/test_copy_format--1.0.sql create mode 100644 src/test/modules/test_copy_format/test_copy_format.c create mode 100644 src/test/modules/test_copy_format/test_copy_format.control diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 5f3697a5f9..6f0db0ae7c 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -32,6 +32,7 @@ #include "parser/parse_coerce.h" #include "parser/parse_collate.h" #include "parser/parse_expr.h" +#include "parser/parse_func.h" #include "parser/parse_relation.h" #include "rewrite/rewriteHandler.h" #include "utils/acl.h" @@ -430,6 +431,69 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from) return COPY_ON_ERROR_STOP; /* keep compiler quiet */ } +/* + * Process the "format" option. + * + * This function checks whether the option value is a built-in format such as + * "text" and "csv" or not. If the option value isn't a built-in format, this + * function finds a COPY format handler that returns a CopyToRoutine. If no + * COPY format handler is found, this function reports an error. + */ +static void +ProcessCopyOptionCustomFormat(ParseState *pstate, + CopyFormatOptions *opts_out, + bool is_from, + DefElem *defel) +{ + char *format; + Oid funcargtypes[1]; + Oid handlerOid = InvalidOid; + Datum datum; + void *routine; + + format = defGetString(defel); + + /* built-in formats */ + if (strcmp(format, "text") == 0) + /* default format */ return; + else if (strcmp(format, "csv") == 0) + { + opts_out->csv_mode = true; + opts_out->to_routine = &CopyToRoutineCSV; + return; + } + else if (strcmp(format, "binary") == 0) + { + opts_out->binary = true; + opts_out->to_routine = &CopyToRoutineBinary; + return; + } + + /* custom format */ + if (!is_from) + { + funcargtypes[0] = INTERNALOID; + handlerOid = LookupFuncName(list_make1(makeString(format)), 1, + funcargtypes, true); + } + if (!OidIsValid(handlerOid)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY format \"%s\" not recognized", format), + parser_errposition(pstate, defel->location))); + + datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from)); + routine = DatumGetPointer(datum); + if (routine == NULL || !IsA(routine, CopyToRoutine)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct", + format, handlerOid), + parser_errposition(pstate, defel->location))); + + opts_out->to_routine = routine; +} + /* * Process the statement option list for COPY. * @@ -481,28 +545,10 @@ ProcessCopyOptions(ParseState *pstate, if (strcmp(defel->defname, "format") == 0) { - char *fmt = defGetString(defel); - if (format_specified) errorConflictingDefElem(defel, pstate); format_specified = true; - if (strcmp(fmt, "text") == 0) - /* default format */ ; - else if (strcmp(fmt, "csv") == 0) - { - opts_out->csv_mode = true; - opts_out->to_routine = &CopyToRoutineCSV; - } - else if (strcmp(fmt, "binary") == 0) - { - opts_out->binary = true; - opts_out->to_routine = &CopyToRoutineBinary; - } - else - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt), - parser_errposition(pstate, defel->location))); + ProcessCopyOptionCustomFormat(pstate, opts_out, is_from, defel); } } /* Extract options except "format" from the statement node tree */ diff --git a/src/backend/nodes/Makefile b/src/backend/nodes/Makefile index 66bbad8e6e..173ee11811 100644 --- a/src/backend/nodes/Makefile +++ b/src/backend/nodes/Makefile @@ -49,6 +49,7 @@ node_headers = \ access/sdir.h \ access/tableam.h \ access/tsmapi.h \ + commands/copyapi.h \ commands/event_trigger.h \ commands/trigger.h \ executor/tuptable.h \ diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl old mode 100644 new mode 100755 index 2f0a59bc87..bd397f45ac --- a/src/backend/nodes/gen_node_support.pl +++ b/src/backend/nodes/gen_node_support.pl @@ -61,6 +61,7 @@ my @all_input_files = qw( access/sdir.h access/tableam.h access/tsmapi.h + commands/copyapi.h commands/event_trigger.h commands/trigger.h executor/tuptable.h @@ -85,6 +86,7 @@ my @nodetag_only_files = qw( access/sdir.h access/tableam.h access/tsmapi.h + commands/copyapi.h commands/event_trigger.h commands/trigger.h executor/tuptable.h diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index a3a991f634..d308780c43 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -373,6 +373,7 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(fdw_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(table_am_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(index_am_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(tsm_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(copy_handler); PSEUDOTYPE_DUMMY_IO_FUNCS(internal); PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement); PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 29af4ce65d..d4e426687c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -7617,6 +7617,12 @@ { oid => '3312', descr => 'I/O', proname => 'tsm_handler_out', prorettype => 'cstring', proargtypes => 'tsm_handler', prosrc => 'tsm_handler_out' }, +{ oid => '8753', descr => 'I/O', + proname => 'copy_handler_in', proisstrict => 'f', prorettype => 'copy_handler', + proargtypes => 'cstring', prosrc => 'copy_handler_in' }, +{ oid => '8754', descr => 'I/O', + proname => 'copy_handler_out', prorettype => 'cstring', + proargtypes => 'copy_handler', prosrc => 'copy_handler_out' }, { oid => '267', descr => 'I/O', proname => 'table_am_handler_in', proisstrict => 'f', prorettype => 'table_am_handler', proargtypes => 'cstring', diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat index d29194da31..2040d5da83 100644 --- a/src/include/catalog/pg_type.dat +++ b/src/include/catalog/pg_type.dat @@ -632,6 +632,12 @@ typcategory => 'P', typinput => 'tsm_handler_in', typoutput => 'tsm_handler_out', typreceive => '-', typsend => '-', typalign => 'i' }, +{ oid => '8752', + descr => 'pseudo-type for the result of a copy to/from method functoin', + typname => 'copy_handler', typlen => '4', typbyval => 't', typtype => 'p', + typcategory => 'P', typinput => 'copy_handler_in', + typoutput => 'copy_handler_out', typreceive => '-', typsend => '-', + typalign => 'i' }, { oid => '269', typname => 'table_am_handler', descr => 'pseudo-type for the result of a table AM handler function', diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index eb68f2fb7b..9c25e1c415 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -29,6 +29,8 @@ typedef void (*CopyToEnd_function) (CopyToState cstate); /* Routines for a COPY TO format implementation. */ typedef struct CopyToRoutine { + NodeTag type; + /* * Called for processing one COPY TO option. This will return false when * the given option is invalid. diff --git a/src/include/nodes/meson.build b/src/include/nodes/meson.build index b665e55b65..103df1a787 100644 --- a/src/include/nodes/meson.build +++ b/src/include/nodes/meson.build @@ -11,6 +11,7 @@ node_support_input_i = [ 'access/sdir.h', 'access/tableam.h', 'access/tsmapi.h', + 'commands/copyapi.h', 'commands/event_trigger.h', 'commands/trigger.h', 'executor/tuptable.h', diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index e32c8925f6..9d57b868d5 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -15,6 +15,7 @@ SUBDIRS = \ spgist_name_ops \ test_bloomfilter \ test_copy_callbacks \ + test_copy_format \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 397e0906e6..d76f2a6003 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -13,6 +13,7 @@ subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_bloomfilter') subdir('test_copy_callbacks') +subdir('test_copy_format') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') diff --git a/src/test/modules/test_copy_format/.gitignore b/src/test/modules/test_copy_format/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_copy_format/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_copy_format/Makefile b/src/test/modules/test_copy_format/Makefile new file mode 100644 index 0000000000..8497f91624 --- /dev/null +++ b/src/test/modules/test_copy_format/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_copy_format/Makefile + +MODULE_big = test_copy_format +OBJS = \ + $(WIN32RES) \ + test_copy_format.o +PGFILEDESC = "test_copy_format - test custom COPY FORMAT" + +EXTENSION = test_copy_format +DATA = test_copy_format--1.0.sql + +REGRESS = test_copy_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_copy_format +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out new file mode 100644 index 0000000000..3a24ae7b97 --- /dev/null +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -0,0 +1,17 @@ +CREATE EXTENSION test_copy_format; +CREATE TABLE public.test (a INT, b INT, c INT); +INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test TO stdout WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +NOTICE: test_copy_format: is_from=false +NOTICE: CopyToProcessOption: "option_before"="before" +NOTICE: CopyToProcessOption: "option_after"="after" +NOTICE: CopyToGetFormat +NOTICE: CopyToStart: natts=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToOneRow: tts_nvalid=3 +NOTICE: CopyToEnd diff --git a/src/test/modules/test_copy_format/meson.build b/src/test/modules/test_copy_format/meson.build new file mode 100644 index 0000000000..4cefe7b709 --- /dev/null +++ b/src/test/modules/test_copy_format/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_copy_format_sources = files( + 'test_copy_format.c', +) + +if host_system == 'windows' + test_copy_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_copy_format', + '--FILEDESC', 'test_copy_format - test custom COPY FORMAT',]) +endif + +test_copy_format = shared_module('test_copy_format', + test_copy_format_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_copy_format + +test_install_data += files( + 'test_copy_format.control', + 'test_copy_format--1.0.sql', +) + +tests += { + 'name': 'test_copy_format', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_copy_format', + ], + }, +} diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql new file mode 100644 index 0000000000..0eb7ed2e11 --- /dev/null +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -0,0 +1,8 @@ +CREATE EXTENSION test_copy_format; +CREATE TABLE public.test (a INT, b INT, c INT); +INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test TO stdout WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); diff --git a/src/test/modules/test_copy_format/test_copy_format--1.0.sql b/src/test/modules/test_copy_format/test_copy_format--1.0.sql new file mode 100644 index 0000000000..d24ea03ce9 --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format--1.0.sql @@ -0,0 +1,8 @@ +/* src/test/modules/test_copy_format/test_copy_format--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_copy_format" to load this file. \quit + +CREATE FUNCTION test_copy_format(internal) + RETURNS copy_handler + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c new file mode 100644 index 0000000000..a2219afcde --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -0,0 +1,77 @@ +/*-------------------------------------------------------------------------- + * + * test_copy_format.c + * Code for testing custom COPY format. + * + * Portions Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_copy_format/test_copy_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/defrem.h" + +PG_MODULE_MAGIC; + +static bool +CopyToProcessOption(CopyToState cstate, DefElem *defel) +{ + ereport(NOTICE, + (errmsg("CopyToProcessOption: \"%s\"=\"%s\"", + defel->defname, defGetString(defel)))); + return true; +} + +static int16 +CopyToGetFormat(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToGetFormat"))); + return 0; +} + +static void +CopyToStart(CopyToState cstate, TupleDesc tupDesc) +{ + ereport(NOTICE, (errmsg("CopyToStart: natts=%d", tupDesc->natts))); +} + +static void +CopyToOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", slot->tts_nvalid))); +} + +static void +CopyToEnd(CopyToState cstate) +{ + ereport(NOTICE, (errmsg("CopyToEnd"))); +} + +static const CopyToRoutine CopyToRoutineTestCopyFormat = { + .type = T_CopyToRoutine, + .CopyToProcessOption = CopyToProcessOption, + .CopyToGetFormat = CopyToGetFormat, + .CopyToStart = CopyToStart, + .CopyToOneRow = CopyToOneRow, + .CopyToEnd = CopyToEnd, +}; + +PG_FUNCTION_INFO_V1(test_copy_format); +Datum +test_copy_format(PG_FUNCTION_ARGS) +{ + bool is_from = PG_GETARG_BOOL(0); + + ereport(NOTICE, + (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false"))); + + if (is_from) + elog(ERROR, "COPY FROM isn't supported yet"); + + PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); +} diff --git a/src/test/modules/test_copy_format/test_copy_format.control b/src/test/modules/test_copy_format/test_copy_format.control new file mode 100644 index 0000000000..f05a636235 --- /dev/null +++ b/src/test/modules/test_copy_format/test_copy_format.control @@ -0,0 +1,4 @@ +comment = 'Test code for custom COPY format' +default_version = '1.0' +module_pathname = '$libdir/test_copy_format' +relocatable = true -- 2.41.0 [application/octet-stream] v8-0003-Export-CopyToStateData.patch (13.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/5-v8-0003-Export-CopyToStateData.patch) download | inline diff: From c3a59753b1157dc8e47e719263f58677acc33178 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 14:54:10 +0900 Subject: [PATCH v8 03/10] Export CopyToStateData It's for custom COPY TO format handlers implemented as extension. This just moves codes. This doesn't change codes except CopyDest enum values. CopyDest enum values such as COPY_FILE are conflicted CopySource enum values defined in copyfrom_internal.h. So COPY_DEST_ prefix instead of COPY_ prefix is used. For example, COPY_FILE is renamed to COPY_DEST_FILE. Note that this change isn't enough to implement a custom COPY TO format handler as extension. We'll do the followings in a subsequent commit: 1. Add an opaque space for custom COPY TO format handler 2. Export CopySendEndOfRow() to flush buffer --- src/backend/commands/copyto.c | 74 +++----------------- src/include/commands/copy.h | 59 ---------------- src/include/commands/copyapi.h | 120 ++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 126 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 6547b7c654..cfc74ee7b1 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -43,64 +43,6 @@ #include "utils/rel.h" #include "utils/snapmgr.h" -/* - * Represents the different dest cases we need to worry about at - * the bottom level - */ -typedef enum CopyDest -{ - COPY_FILE, /* to file (or a piped program) */ - COPY_FRONTEND, /* to frontend */ - COPY_CALLBACK, /* to callback function */ -} CopyDest; - -/* - * This struct contains all the state variables used throughout a COPY TO - * operation. - * - * Multi-byte encodings: all supported client-side encodings encode multi-byte - * characters by having the first byte's high bit set. Subsequent bytes of the - * character can have the high bit not set. When scanning data in such an - * encoding to look for a match to a single-byte (ie ASCII) character, we must - * use the full pg_encoding_mblen() machinery to skip over multibyte - * characters, else we might find a false match to a trailing byte. In - * supported server encodings, there is no possibility of a false match, and - * it's faster to make useless comparisons to trailing bytes than it is to - * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true - * when we have to do it the hard way. - */ -typedef struct CopyToStateData -{ - /* low-level state data */ - CopyDest copy_dest; /* type of copy source/destination */ - FILE *copy_file; /* used if copy_dest == COPY_FILE */ - StringInfo fe_msgbuf; /* used for all dests during COPY TO */ - - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy to */ - QueryDesc *queryDesc; /* executable query to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDOUT */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_dest_cb data_dest_cb; /* function for writing data */ - - CopyFormatOptions opts; - Node *whereClause; /* WHERE condition (or NULL) */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - FmgrInfo *out_functions; /* lookup info for output functions */ - MemoryContext rowcontext; /* per-row evaluation context */ - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyToStateData; - /* DestReceiver for COPY (query) TO */ typedef struct { @@ -160,7 +102,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) { switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: /* Default line termination depends on platform */ #ifndef WIN32 CopySendChar(cstate, '\n'); @@ -168,7 +110,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) CopySendString(cstate, "\r\n"); #endif break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); break; @@ -419,7 +361,7 @@ SendCopyBegin(CopyToState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_dest = COPY_FRONTEND; + cstate->copy_dest = COPY_DEST_FRONTEND; } static void @@ -466,7 +408,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { - case COPY_FILE: + case COPY_DEST_FILE: if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1, cstate->copy_file) != 1 || ferror(cstate->copy_file)) @@ -500,11 +442,11 @@ CopySendEndOfRow(CopyToState cstate) errmsg("could not write to COPY file: %m"))); } break; - case COPY_FRONTEND: + case COPY_DEST_FRONTEND: /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); break; - case COPY_CALLBACK: + case COPY_DEST_CALLBACK: cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len); break; } @@ -877,12 +819,12 @@ BeginCopyTo(ParseState *pstate, /* See Multibyte encoding comment above */ cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); - cstate->copy_dest = COPY_FILE; /* default */ + cstate->copy_dest = COPY_DEST_FILE; /* default */ if (data_dest_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_dest = COPY_CALLBACK; + cstate->copy_dest = COPY_DEST_CALLBACK; cstate->data_dest_cb = data_dest_cb; } else if (pipe) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 34bea880ca..b3f4682f95 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,69 +20,10 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -/* - * Represents whether a header line should be present, and whether it must - * match the actual names (which implies "true"). - */ -typedef enum CopyHeaderChoice -{ - COPY_HEADER_FALSE = 0, - COPY_HEADER_TRUE, - COPY_HEADER_MATCH, -} CopyHeaderChoice; - -/* - * Represents where to save input processing errors. More values to be added - * in the future. - */ -typedef enum CopyOnErrorChoice -{ - COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */ - COPY_ON_ERROR_IGNORE, /* ignore errors */ -} CopyOnErrorChoice; - -/* - * A struct to hold COPY options, in a parsed form. All of these are related - * to formatting, except for 'freeze', which doesn't really belong here, but - * it's expedient to parse it along with all the other options. - */ -typedef struct CopyFormatOptions -{ - /* parameters from the COPY command */ - int file_encoding; /* file or remote side's character encoding, - * -1 if not specified */ - bool binary; /* binary format? */ - bool freeze; /* freeze rows on loading? */ - bool csv_mode; /* Comma Separated Value format? */ - CopyHeaderChoice header_line; /* header line? */ - char *null_print; /* NULL marker string (server encoding!) */ - int null_print_len; /* length of same */ - char *null_print_client; /* same converted to file encoding */ - char *default_print; /* DEFAULT marker string */ - int default_print_len; /* length of same */ - char *delim; /* column delimiter (must be 1 byte) */ - char *quote; /* CSV quote char (must be 1 byte) */ - char *escape; /* CSV escape char (must be 1 byte) */ - List *force_quote; /* list of column names */ - bool force_quote_all; /* FORCE_QUOTE *? */ - bool *force_quote_flags; /* per-column CSV FQ flags */ - List *force_notnull; /* list of column names */ - bool force_notnull_all; /* FORCE_NOT_NULL *? */ - bool *force_notnull_flags; /* per-column CSV FNN flags */ - List *force_null; /* list of column names */ - bool force_null_all; /* FORCE_NULL *? */ - bool *force_null_flags; /* per-column CSV FN flags */ - bool convert_selectively; /* do selective binary conversion? */ - CopyOnErrorChoice on_error; /* what to do when error happened */ - List *convert_select; /* list of column names (can be NIL) */ - CopyToRoutine *to_routine; /* callback routines for COPY TO */ -} CopyFormatOptions; - /* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); -typedef void (*copy_data_dest_cb) (void *data, int len); extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 9c25e1c415..a869d78d72 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,10 +14,10 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "executor/execdesc.h" #include "executor/tuptable.h" #include "nodes/parsenodes.h" -/* This is private in commands/copyto.c */ typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); @@ -58,4 +58,122 @@ extern CopyToRoutine CopyToRoutineText; extern CopyToRoutine CopyToRoutineCSV; extern CopyToRoutine CopyToRoutineBinary; +/* + * Represents whether a header line should be present, and whether it must + * match the actual names (which implies "true"). + */ +typedef enum CopyHeaderChoice +{ + COPY_HEADER_FALSE = 0, + COPY_HEADER_TRUE, + COPY_HEADER_MATCH, +} CopyHeaderChoice; + +/* + * Represents where to save input processing errors. More values to be added + * in the future. + */ +typedef enum CopyOnErrorChoice +{ + COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */ + COPY_ON_ERROR_IGNORE, /* ignore errors */ +} CopyOnErrorChoice; + +/* + * A struct to hold COPY options, in a parsed form. All of these are related + * to formatting, except for 'freeze', which doesn't really belong here, but + * it's expedient to parse it along with all the other options. + */ +typedef struct CopyFormatOptions +{ + /* parameters from the COPY command */ + int file_encoding; /* file or remote side's character encoding, + * -1 if not specified */ + bool binary; /* binary format? */ + bool freeze; /* freeze rows on loading? */ + bool csv_mode; /* Comma Separated Value format? */ + CopyHeaderChoice header_line; /* header line? */ + char *null_print; /* NULL marker string (server encoding!) */ + int null_print_len; /* length of same */ + char *null_print_client; /* same converted to file encoding */ + char *default_print; /* DEFAULT marker string */ + int default_print_len; /* length of same */ + char *delim; /* column delimiter (must be 1 byte) */ + char *quote; /* CSV quote char (must be 1 byte) */ + char *escape; /* CSV escape char (must be 1 byte) */ + List *force_quote; /* list of column names */ + bool force_quote_all; /* FORCE_QUOTE *? */ + bool *force_quote_flags; /* per-column CSV FQ flags */ + List *force_notnull; /* list of column names */ + bool force_notnull_all; /* FORCE_NOT_NULL *? */ + bool *force_notnull_flags; /* per-column CSV FNN flags */ + List *force_null; /* list of column names */ + bool force_null_all; /* FORCE_NULL *? */ + bool *force_null_flags; /* per-column CSV FN flags */ + bool convert_selectively; /* do selective binary conversion? */ + CopyOnErrorChoice on_error; /* what to do when error happened */ + List *convert_select; /* list of column names (can be NIL) */ + CopyToRoutine *to_routine; /* callback routines for COPY TO */ +} CopyFormatOptions; + +/* + * Represents the different dest cases we need to worry about at + * the bottom level + */ +typedef enum CopyDest +{ + COPY_DEST_FILE, /* to file (or a piped program) */ + COPY_DEST_FRONTEND, /* to frontend */ + COPY_DEST_CALLBACK, /* to callback function */ +} CopyDest; + +typedef void (*copy_data_dest_cb) (void *data, int len); + +/* + * This struct contains all the state variables used throughout a COPY TO + * operation. + * + * Multi-byte encodings: all supported client-side encodings encode multi-byte + * characters by having the first byte's high bit set. Subsequent bytes of the + * character can have the high bit not set. When scanning data in such an + * encoding to look for a match to a single-byte (ie ASCII) character, we must + * use the full pg_encoding_mblen() machinery to skip over multibyte + * characters, else we might find a false match to a trailing byte. In + * supported server encodings, there is no possibility of a false match, and + * it's faster to make useless comparisons to trailing bytes than it is to + * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true + * when we have to do it the hard way. + */ +typedef struct CopyToStateData +{ + /* low-level state data */ + CopyDest copy_dest; /* type of copy source/destination */ + FILE *copy_file; /* used if copy_dest == COPY_FILE */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO */ + + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy to */ + QueryDesc *queryDesc; /* executable query to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDOUT */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_dest_cb data_dest_cb; /* function for writing data */ + + CopyFormatOptions opts; + Node *whereClause; /* WHERE condition (or NULL) */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + FmgrInfo *out_functions; /* lookup info for output functions */ + MemoryContext rowcontext; /* per-row evaluation context */ + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyToStateData; + #endif /* COPYAPI_H */ -- 2.41.0 [application/octet-stream] v8-0005-Extract-COPY-FROM-format-implementations.patch (24.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/6-v8-0005-Extract-COPY-FROM-format-implementations.patch) download | inline diff: From 781955f19ad27cdd66748be539bf45cf1b925856 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Tue, 23 Jan 2024 17:21:23 +0900 Subject: [PATCH v8 05/10] Extract COPY FROM format implementations This doesn't change the current behavior. This just introduces CopyFromRoutine, which just has function pointers of format implementation like TupleTableSlotOps, and use it for existing "text", "csv" and "binary" format implementations. Note that CopyFromRoutine can't be used from extensions yet because CopyRead*() aren't exported yet. Extensions can't read data from a source without CopyRead*(). They will be exported by subsequent patches. --- src/backend/commands/copy.c | 3 + src/backend/commands/copyfrom.c | 216 +++++++++++---- src/backend/commands/copyfromparse.c | 326 ++++++++++++----------- src/include/commands/copy.h | 3 - src/include/commands/copyapi.h | 44 +++ src/include/commands/copyfrom_internal.h | 4 + 6 files changed, 391 insertions(+), 205 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 6f0db0ae7c..ec6dfff8ab 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -459,12 +459,14 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, else if (strcmp(format, "csv") == 0) { opts_out->csv_mode = true; + opts_out->from_routine = &CopyFromRoutineCSV; opts_out->to_routine = &CopyToRoutineCSV; return; } else if (strcmp(format, "binary") == 0) { opts_out->binary = true; + opts_out->from_routine = &CopyFromRoutineBinary; opts_out->to_routine = &CopyToRoutineBinary; return; } @@ -533,6 +535,7 @@ ProcessCopyOptions(ParseState *pstate, opts_out->file_encoding = -1; /* Text is the default format. */ + opts_out->from_routine = &CopyFromRoutineText; opts_out->to_routine = &CopyToRoutineText; /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index fb3d4d9296..d556ebb5d6 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -108,6 +108,170 @@ static char *limit_printout_length(const char *str); static void ClosePipeFromProgram(CopyFromState cstate); + +/* + * CopyFromRoutine implementations. + */ + +/* + * CopyFromRoutine implementation for "text" and "csv". CopyFromText*() + * refer cstate->opts.csv_mode and change their behavior. We can split this + * implementation and stop referring cstate->opts.csv_mode later. + */ + +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may + * move the code to here later. */ +static bool +CopyFromTextProcessOption(CopyFromState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyFromTextGetFormat(CopyFromState cstate) +{ + return 0; +} + +static void +CopyFromTextStart(CopyFromState cstate, TupleDesc tupDesc) +{ + AttrNumber num_phys_attrs = tupDesc->natts; + AttrNumber attr_count; + + /* + * If encoding conversion is needed, we need another buffer to hold the + * converted input data. Otherwise, we can just point input_buf to the + * same buffer as raw_buf. + */ + if (cstate->need_transcoding) + { + cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); + cstate->input_buf_index = cstate->input_buf_len = 0; + } + else + cstate->input_buf = cstate->raw_buf; + cstate->input_reached_eof = false; + + initStringInfo(&cstate->line_buf); + + /* + * Pick up the required catalog information for each attribute in the + * relation, including the input function, the element type (to pass to + * the input function). + */ + cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); + for (int attnum = 1; attnum <= num_phys_attrs; attnum++) + { + Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1); + Oid in_func_oid; + + /* We don't need info for dropped attributes */ + if (att->attisdropped) + continue; + + /* Fetch the input function and typioparam info */ + getTypeInputInfo(att->atttypid, + &in_func_oid, &cstate->typioparams[attnum - 1]); + fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]); + } + + /* create workspace for CopyReadAttributes results */ + attr_count = list_length(cstate->attnumlist); + cstate->max_fields = attr_count; + cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); +} + +static void +CopyFromTextEnd(CopyFromState cstate) +{ +} + +/* + * CopyFromRoutine implementation for "binary". + */ + +/* All "binary" options are parsed in ProcessCopyOptions(). We may move the + * code to here later. */ +static bool +CopyFromBinaryProcessOption(CopyFromState cstate, DefElem *defel) +{ + return false; +} + +static int16 +CopyFromBinaryGetFormat(CopyFromState cstate) +{ + return 1; +} + +static void +CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc) +{ + AttrNumber num_phys_attrs = tupDesc->natts; + + /* + * Pick up the required catalog information for each attribute in the + * relation, including the input function, the element type (to pass to + * the input function). + */ + cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); + cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); + for (int attnum = 1; attnum <= num_phys_attrs; attnum++) + { + Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1); + Oid in_func_oid; + + /* We don't need info for dropped attributes */ + if (att->attisdropped) + continue; + + /* Fetch the input function and typioparam info */ + getTypeBinaryInputInfo(att->atttypid, + &in_func_oid, &cstate->typioparams[attnum - 1]); + fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]); + } + + /* Read and verify binary header */ + ReceiveCopyBinaryHeader(cstate); +} + +static void +CopyFromBinaryEnd(CopyFromState cstate) +{ +} + +CopyFromRoutine CopyFromRoutineText = { + .CopyFromProcessOption = CopyFromTextProcessOption, + .CopyFromGetFormat = CopyFromTextGetFormat, + .CopyFromStart = CopyFromTextStart, + .CopyFromOneRow = CopyFromTextOneRow, + .CopyFromEnd = CopyFromTextEnd, +}; + +/* + * We can use the same CopyFromRoutine for both of "text" and "csv" because + * CopyFromText*() refer cstate->opts.csv_mode and change their behavior. We can + * split the implementations and stop referring cstate->opts.csv_mode later. + */ +CopyFromRoutine CopyFromRoutineCSV = { + .CopyFromProcessOption = CopyFromTextProcessOption, + .CopyFromGetFormat = CopyFromTextGetFormat, + .CopyFromStart = CopyFromTextStart, + .CopyFromOneRow = CopyFromTextOneRow, + .CopyFromEnd = CopyFromTextEnd, +}; + +CopyFromRoutine CopyFromRoutineBinary = { + .CopyFromProcessOption = CopyFromBinaryProcessOption, + .CopyFromGetFormat = CopyFromBinaryGetFormat, + .CopyFromStart = CopyFromBinaryStart, + .CopyFromOneRow = CopyFromBinaryOneRow, + .CopyFromEnd = CopyFromBinaryEnd, +}; + + /* * error context callback for COPY FROM * @@ -1384,9 +1548,6 @@ BeginCopyFrom(ParseState *pstate, TupleDesc tupDesc; AttrNumber num_phys_attrs, num_defaults; - FmgrInfo *in_functions; - Oid *typioparams; - Oid in_func_oid; int *defmap; ExprState **defexprs; MemoryContext oldcontext; @@ -1571,25 +1732,6 @@ BeginCopyFrom(ParseState *pstate, cstate->raw_buf_index = cstate->raw_buf_len = 0; cstate->raw_reached_eof = false; - if (!cstate->opts.binary) - { - /* - * If encoding conversion is needed, we need another buffer to hold - * the converted input data. Otherwise, we can just point input_buf - * to the same buffer as raw_buf. - */ - if (cstate->need_transcoding) - { - cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1); - cstate->input_buf_index = cstate->input_buf_len = 0; - } - else - cstate->input_buf = cstate->raw_buf; - cstate->input_reached_eof = false; - - initStringInfo(&cstate->line_buf); - } - initStringInfo(&cstate->attribute_buf); /* Assign range table and rteperminfos, we'll need them in CopyFrom. */ @@ -1608,8 +1750,6 @@ BeginCopyFrom(ParseState *pstate, * the input function), and info about defaults and constraints. (Which * input function we use depends on text/binary format choice.) */ - in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); - typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid)); defmap = (int *) palloc(num_phys_attrs * sizeof(int)); defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *)); @@ -1621,15 +1761,6 @@ BeginCopyFrom(ParseState *pstate, if (att->attisdropped) continue; - /* Fetch the input function and typioparam info */ - if (cstate->opts.binary) - getTypeBinaryInputInfo(att->atttypid, - &in_func_oid, &typioparams[attnum - 1]); - else - getTypeInputInfo(att->atttypid, - &in_func_oid, &typioparams[attnum - 1]); - fmgr_info(in_func_oid, &in_functions[attnum - 1]); - /* Get default info if available */ defexprs[attnum - 1] = NULL; @@ -1689,8 +1820,6 @@ BeginCopyFrom(ParseState *pstate, cstate->bytes_processed = 0; /* We keep those variables in cstate. */ - cstate->in_functions = in_functions; - cstate->typioparams = typioparams; cstate->defmap = defmap; cstate->defexprs = defexprs; cstate->volatile_defexprs = volatile_defexprs; @@ -1763,20 +1892,7 @@ BeginCopyFrom(ParseState *pstate, pgstat_progress_update_multi_param(3, progress_cols, progress_vals); - if (cstate->opts.binary) - { - /* Read and verify binary header */ - ReceiveCopyBinaryHeader(cstate); - } - - /* create workspace for CopyReadAttributes results */ - if (!cstate->opts.binary) - { - AttrNumber attr_count = list_length(cstate->attnumlist); - - cstate->max_fields = attr_count; - cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *)); - } + cstate->opts.from_routine->CopyFromStart(cstate, tupDesc); MemoryContextSwitchTo(oldcontext); @@ -1789,6 +1905,8 @@ BeginCopyFrom(ParseState *pstate, void EndCopyFrom(CopyFromState cstate) { + cstate->opts.from_routine->CopyFromEnd(cstate); + /* No COPY FROM related resources except memory. */ if (cstate->is_program) { diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 7cacd0b752..49632f75e4 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -172,7 +172,7 @@ ReceiveCopyBegin(CopyFromState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = cstate->opts.from_routine->CopyFromGetFormat(cstate); int i; pq_beginmessage(&buf, PqMsg_CopyInResponse); @@ -840,199 +840,219 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) return true; } -/* - * Read next tuple from file for COPY FROM. Return false if no more tuples. - * - * 'econtext' is used to evaluate default expression for each column that is - * either not read from the file or is using the DEFAULT option of COPY FROM. - * It can be NULL when no default values are used, i.e. when all columns are - * read from the file, and DEFAULT option is unset. - * - * 'values' and 'nulls' arrays must be the same length as columns of the - * relation passed to BeginCopyFrom. This function fills the arrays. - */ bool -NextCopyFrom(CopyFromState cstate, ExprContext *econtext, - Datum *values, bool *nulls) +CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { TupleDesc tupDesc; - AttrNumber num_phys_attrs, - attr_count, - num_defaults = cstate->num_defaults; + AttrNumber attr_count; FmgrInfo *in_functions = cstate->in_functions; Oid *typioparams = cstate->typioparams; - int i; - int *defmap = cstate->defmap; ExprState **defexprs = cstate->defexprs; + char **field_strings; + ListCell *cur; + int fldct; + int fieldno; + char *string; tupDesc = RelationGetDescr(cstate->rel); - num_phys_attrs = tupDesc->natts; attr_count = list_length(cstate->attnumlist); - /* Initialize all values for row to NULL */ - MemSet(values, 0, num_phys_attrs * sizeof(Datum)); - MemSet(nulls, true, num_phys_attrs * sizeof(bool)); - MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); + /* read raw fields in the next line */ + if (!NextCopyFromRawFields(cstate, &field_strings, &fldct)) + return false; - if (!cstate->opts.binary) - { - char **field_strings; - ListCell *cur; - int fldct; - int fieldno; - char *string; + /* check for overflowing fields */ + if (attr_count > 0 && fldct > attr_count) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("extra data after last expected column"))); - /* read raw fields in the next line */ - if (!NextCopyFromRawFields(cstate, &field_strings, &fldct)) - return false; + fieldno = 0; - /* check for overflowing fields */ - if (attr_count > 0 && fldct > attr_count) + /* Loop to read the user attributes on the line. */ + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (fieldno >= fldct) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("extra data after last expected column"))); + errmsg("missing data for column \"%s\"", + NameStr(att->attname)))); + string = field_strings[fieldno++]; - fieldno = 0; - - /* Loop to read the user attributes on the line. */ - foreach(cur, cstate->attnumlist) + if (cstate->convert_select_flags && + !cstate->convert_select_flags[m]) { - int attnum = lfirst_int(cur); - int m = attnum - 1; - Form_pg_attribute att = TupleDescAttr(tupDesc, m); - - if (fieldno >= fldct) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("missing data for column \"%s\"", - NameStr(att->attname)))); - string = field_strings[fieldno++]; - - if (cstate->convert_select_flags && - !cstate->convert_select_flags[m]) - { - /* ignore input field, leaving column as NULL */ - continue; - } + /* ignore input field, leaving column as NULL */ + continue; + } - if (cstate->opts.csv_mode) + if (cstate->opts.csv_mode) + { + if (string == NULL && + cstate->opts.force_notnull_flags[m]) { - if (string == NULL && - cstate->opts.force_notnull_flags[m]) - { - /* - * FORCE_NOT_NULL option is set and column is NULL - - * convert it to the NULL string. - */ - string = cstate->opts.null_print; - } - else if (string != NULL && cstate->opts.force_null_flags[m] - && strcmp(string, cstate->opts.null_print) == 0) - { - /* - * FORCE_NULL option is set and column matches the NULL - * string. It must have been quoted, or otherwise the - * string would already have been set to NULL. Convert it - * to NULL as specified. - */ - string = NULL; - } + /* + * FORCE_NOT_NULL option is set and column is NULL - convert + * it to the NULL string. + */ + string = cstate->opts.null_print; } - - cstate->cur_attname = NameStr(att->attname); - cstate->cur_attval = string; - - if (string != NULL) - nulls[m] = false; - - if (cstate->defaults[m]) + else if (string != NULL && cstate->opts.force_null_flags[m] + && strcmp(string, cstate->opts.null_print) == 0) { /* - * The caller must supply econtext and have switched into the - * per-tuple memory context in it. + * FORCE_NULL option is set and column matches the NULL + * string. It must have been quoted, or otherwise the string + * would already have been set to NULL. Convert it to NULL as + * specified. */ - Assert(econtext != NULL); - Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); - - values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); + string = NULL; } + } + + cstate->cur_attname = NameStr(att->attname); + cstate->cur_attval = string; + + if (string != NULL) + nulls[m] = false; + if (cstate->defaults[m]) + { /* - * If ON_ERROR is specified with IGNORE, skip rows with soft - * errors + * The caller must supply econtext and have switched into the + * per-tuple memory context in it. */ - else if (!InputFunctionCallSafe(&in_functions[m], - string, - typioparams[m], - att->atttypmod, - (Node *) cstate->escontext, - &values[m])) - { - cstate->num_errors++; - return true; - } + Assert(econtext != NULL); + Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; + values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); } - Assert(fieldno == attr_count); + /* + * If ON_ERROR is specified with IGNORE, skip rows with soft errors + */ + else if (!InputFunctionCallSafe(&in_functions[m], + string, + typioparams[m], + att->atttypmod, + (Node *) cstate->escontext, + &values[m])) + { + cstate->num_errors++; + return true; + } + + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; } - else - { - /* binary */ - int16 fld_count; - ListCell *cur; - cstate->cur_lineno++; + Assert(fieldno == attr_count); - if (!CopyGetInt16(cstate, &fld_count)) - { - /* EOF detected (end of file, or protocol-level EOF) */ - return false; - } + return true; +} - if (fld_count == -1) - { - /* - * Received EOF marker. Wait for the protocol-level EOF, and - * complain if it doesn't come immediately. In COPY FROM STDIN, - * this ensures that we correctly handle CopyFail, if client - * chooses to send that now. When copying from file, we could - * ignore the rest of the file like in text mode, but we choose to - * be consistent with the COPY FROM STDIN case. - */ - char dummy; +bool +CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + TupleDesc tupDesc; + AttrNumber attr_count; + FmgrInfo *in_functions = cstate->in_functions; + Oid *typioparams = cstate->typioparams; + int16 fld_count; + ListCell *cur; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("received copy data after EOF marker"))); - return false; - } + tupDesc = RelationGetDescr(cstate->rel); + attr_count = list_length(cstate->attnumlist); - if (fld_count != attr_count) + cstate->cur_lineno++; + + if (!CopyGetInt16(cstate, &fld_count)) + { + /* EOF detected (end of file, or protocol-level EOF) */ + return false; + } + + if (fld_count == -1) + { + /* + * Received EOF marker. Wait for the protocol-level EOF, and complain + * if it doesn't come immediately. In COPY FROM STDIN, this ensures + * that we correctly handle CopyFail, if client chooses to send that + * now. When copying from file, we could ignore the rest of the file + * like in text mode, but we choose to be consistent with the COPY + * FROM STDIN case. + */ + char dummy; + + if (CopyReadBinaryData(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("row field count is %d, expected %d", - (int) fld_count, attr_count))); + errmsg("received copy data after EOF marker"))); + return false; + } - foreach(cur, cstate->attnumlist) - { - int attnum = lfirst_int(cur); - int m = attnum - 1; - Form_pg_attribute att = TupleDescAttr(tupDesc, m); - - cstate->cur_attname = NameStr(att->attname); - values[m] = CopyReadBinaryAttribute(cstate, - &in_functions[m], - typioparams[m], - att->atttypmod, - &nulls[m]); - cstate->cur_attname = NULL; - } + if (fld_count != attr_count) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("row field count is %d, expected %d", + (int) fld_count, attr_count))); + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + cstate->cur_attname = NameStr(att->attname); + values[m] = CopyReadBinaryAttribute(cstate, + &in_functions[m], + typioparams[m], + att->atttypmod, + &nulls[m]); + cstate->cur_attname = NULL; } + return true; +} + +/* + * Read next tuple from file for COPY FROM. Return false if no more tuples. + * + * 'econtext' is used to evaluate default expression for each column that is + * either not read from the file or is using the DEFAULT option of COPY FROM. + * It can be NULL when no default values are used, i.e. when all columns are + * read from the file, and DEFAULT option is unset. + * + * 'values' and 'nulls' arrays must be the same length as columns of the + * relation passed to BeginCopyFrom. This function fills the arrays. + */ +bool +NextCopyFrom(CopyFromState cstate, ExprContext *econtext, + Datum *values, bool *nulls) +{ + TupleDesc tupDesc; + AttrNumber num_phys_attrs, + num_defaults = cstate->num_defaults; + int i; + int *defmap = cstate->defmap; + ExprState **defexprs = cstate->defexprs; + + tupDesc = RelationGetDescr(cstate->rel); + num_phys_attrs = tupDesc->natts; + + /* Initialize all values for row to NULL */ + MemSet(values, 0, num_phys_attrs * sizeof(Datum)); + MemSet(nulls, true, num_phys_attrs * sizeof(bool)); + MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); + + if (!cstate->opts.from_routine->CopyFromOneRow(cstate, econtext, values, + nulls)) + return false; + /* * Now compute and insert any defaults available for the columns not * provided by the input data. Anything not processed here or above will diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index b3f4682f95..df29d42555 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,9 +20,6 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -/* This is private in commands/copyfrom.c */ -typedef struct CopyFromStateData *CopyFromState; - typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index ffad433a21..323e4705d2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -18,6 +18,49 @@ #include "executor/tuptable.h" #include "nodes/parsenodes.h" +/* This is private in commands/copyfrom.c */ +typedef struct CopyFromStateData *CopyFromState; + +typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); +typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate); +typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc); +typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); +typedef void (*CopyFromEnd_function) (CopyFromState cstate); + +/* Routines for a COPY FROM format implementation. */ +typedef struct CopyFromRoutine +{ + /* + * Called for processing one COPY FROM option. This will return false when + * the given option is invalid. + */ + CopyFromProcessOption_function CopyFromProcessOption; + + /* + * Called when COPY FROM is started. This will return a format as int16 + * value. It's used for the CopyInResponse message. + */ + CopyFromGetFormat_function CopyFromGetFormat; + + /* + * Called when COPY FROM is started. This will initialize something and + * receive a header. + */ + CopyFromStart_function CopyFromStart; + + /* Copy one row. It returns false if no more tuples. */ + CopyFromOneRow_function CopyFromOneRow; + + /* Called when COPY FROM is ended. This will finalize something. */ + CopyFromEnd_function CopyFromEnd; +} CopyFromRoutine; + +/* Built-in CopyFromRoutine for "text", "csv" and "binary". */ +extern CopyFromRoutine CopyFromRoutineText; +extern CopyFromRoutine CopyFromRoutineCSV; +extern CopyFromRoutine CopyFromRoutineBinary; + + typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); @@ -113,6 +156,7 @@ typedef struct CopyFormatOptions bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ List *convert_select; /* list of column names (can be NIL) */ + CopyFromRoutine *from_routine; /* callback routines for COPY FROM */ CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index cad52fcc78..921c1513f7 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -183,4 +183,8 @@ typedef struct CopyFromStateData extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); +extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); +extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); + + #endif /* COPYFROM_INTERNAL_H */ -- 2.41.0 [application/octet-stream] v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch (7.6K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/7-v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch) download | inline diff: From f0a8151feff44823881c3c4e1e7aca4f9bd690d5 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Sat, 27 Jan 2024 09:53:31 +0800 Subject: [PATCH v8 09/10] change CopyToGetFormat to CopyToSendCopyBegin and export more api Signed-off-by: Zhao Junwang <[email protected]> --- src/backend/commands/copyto.c | 65 ++++++++++--------- src/include/commands/copyapi.h | 12 ++-- .../test_copy_format/test_copy_format.c | 7 +- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b5d8678394..e2a4964015 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -66,11 +66,6 @@ static void CopyAttributeOutCSV(CopyToState cstate, const char *string, /* Low-level communications functions */ static void SendCopyBegin(CopyToState cstate); static void SendCopyEnd(CopyToState cstate); -static void CopySendData(CopyToState cstate, const void *databuf, int datasize); -static void CopySendString(CopyToState cstate, const char *str); -static void CopySendChar(CopyToState cstate, char c); -static void CopySendInt32(CopyToState cstate, int32 val); -static void CopySendInt16(CopyToState cstate, int16 val); /* * CopyToRoutine implementations. @@ -90,10 +85,20 @@ CopyToTextProcessOption(CopyToState cstate, DefElem *defel) return false; } -static int16 -CopyToTextGetFormat(CopyToState cstate) +static void +CopyToTextSendCopyBegin(CopyToState cstate) { - return 0; + StringInfoData buf; + int natts = list_length(cstate->attnumlist); + int16 format = 0; + int i; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + pq_sendint16(&buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(&buf, format); /* per-column formats */ + pq_endmessage(&buf); } static void @@ -230,10 +235,20 @@ CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel) return false; } -static int16 -CopyToBinaryGetFormat(CopyToState cstate) +static void +CopyToBinarySendCopyBegin(CopyToState cstate) { - return 1; + StringInfoData buf; + int natts = list_length(cstate->attnumlist); + int16 format = 1; + int i; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + pq_sendint16(&buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(&buf, format); /* per-column formats */ + pq_endmessage(&buf); } static void @@ -315,7 +330,7 @@ CopyToBinaryEnd(CopyToState cstate) CopyToRoutine CopyToRoutineText = { .CopyToProcessOption = CopyToTextProcessOption, - .CopyToGetFormat = CopyToTextGetFormat, + .CopyToSendCopyBegin = CopyToTextSendCopyBegin, .CopyToStart = CopyToTextStart, .CopyToOneRow = CopyToTextOneRow, .CopyToEnd = CopyToTextEnd, @@ -328,7 +343,7 @@ CopyToRoutine CopyToRoutineText = { */ CopyToRoutine CopyToRoutineCSV = { .CopyToProcessOption = CopyToTextProcessOption, - .CopyToGetFormat = CopyToTextGetFormat, + .CopyToSendCopyBegin = CopyToTextSendCopyBegin, .CopyToStart = CopyToTextStart, .CopyToOneRow = CopyToTextOneRow, .CopyToEnd = CopyToTextEnd, @@ -336,7 +351,7 @@ CopyToRoutine CopyToRoutineCSV = { CopyToRoutine CopyToRoutineBinary = { .CopyToProcessOption = CopyToBinaryProcessOption, - .CopyToGetFormat = CopyToBinaryGetFormat, + .CopyToSendCopyBegin = CopyToBinarySendCopyBegin, .CopyToStart = CopyToBinaryStart, .CopyToOneRow = CopyToBinaryOneRow, .CopyToEnd = CopyToBinaryEnd, @@ -349,17 +364,7 @@ CopyToRoutine CopyToRoutineBinary = { static void SendCopyBegin(CopyToState cstate) { - StringInfoData buf; - int natts = list_length(cstate->attnumlist); - int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate); - int i; - - pq_beginmessage(&buf, PqMsg_CopyOutResponse); - pq_sendbyte(&buf, format); /* overall format */ - pq_sendint16(&buf, natts); - for (i = 0; i < natts; i++) - pq_sendint16(&buf, format); /* per-column formats */ - pq_endmessage(&buf); + cstate->opts.to_routine->CopyToSendCopyBegin(cstate); cstate->copy_dest = COPY_DEST_FRONTEND; } @@ -382,19 +387,19 @@ SendCopyEnd(CopyToState cstate) * NB: no data conversion is applied by these functions *---------- */ -static void +void CopySendData(CopyToState cstate, const void *databuf, int datasize) { appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize); } -static void +void CopySendString(CopyToState cstate, const char *str) { appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str)); } -static void +void CopySendChar(CopyToState cstate, char c) { appendStringInfoCharMacro(cstate->fe_msgbuf, c); @@ -464,7 +469,7 @@ CopyToStateFlush(CopyToState cstate) /* * CopySendInt32 sends an int32 in network byte order */ -static inline void +inline void CopySendInt32(CopyToState cstate, int32 val) { uint32 buf; @@ -476,7 +481,7 @@ CopySendInt32(CopyToState cstate, int32 val) /* * CopySendInt16 sends an int16 in network byte order */ -static inline void +inline void CopySendInt16(CopyToState cstate, int16 val) { uint16 buf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 22accc83ab..0a05b24c54 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -67,7 +67,7 @@ extern CopyFromRoutine CopyFromRoutineBinary; typedef struct CopyToStateData *CopyToState; typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel); -typedef int16 (*CopyToGetFormat_function) (CopyToState cstate); +typedef void (*CopyToSendCopyBegin_function) (CopyToState cstate); typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc); typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot); typedef void (*CopyToEnd_function) (CopyToState cstate); @@ -84,10 +84,9 @@ typedef struct CopyToRoutine CopyToProcessOption_function CopyToProcessOption; /* - * Called when COPY TO is started. This will return a format as int16 - * value. It's used for the CopyOutResponse message. + * Called when COPY TO is started. */ - CopyToGetFormat_function CopyToGetFormat; + CopyToSendCopyBegin_function CopyToSendCopyBegin; /* Called when COPY TO is started. This will send a header. */ CopyToStart_function CopyToStart; @@ -384,6 +383,11 @@ typedef struct CopyToStateData void *opaque; /* private space */ } CopyToStateData; +extern void CopySendData(CopyToState cstate, const void *databuf, int datasize); +extern void CopySendString(CopyToState cstate, const char *str); +extern void CopySendChar(CopyToState cstate, char c); +extern void CopySendInt32(CopyToState cstate, int32 val); +extern void CopySendInt16(CopyToState cstate, int16 val); extern void CopyToStateFlush(CopyToState cstate); #endif /* COPYAPI_H */ diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 5e1b40e881..d833f22bbf 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -71,11 +71,10 @@ CopyToProcessOption(CopyToState cstate, DefElem *defel) return true; } -static int16 -CopyToGetFormat(CopyToState cstate) +static void +CopyToSendCopyBegin(CopyToState cstate) { ereport(NOTICE, (errmsg("CopyToGetFormat"))); - return 0; } static void @@ -99,7 +98,7 @@ CopyToEnd(CopyToState cstate) static const CopyToRoutine CopyToRoutineTestCopyFormat = { .type = T_CopyToRoutine, .CopyToProcessOption = CopyToProcessOption, - .CopyToGetFormat = CopyToGetFormat, + .CopyToSendCopyBegin = CopyToSendCopyBegin, .CopyToStart = CopyToStart, .CopyToOneRow = CopyToOneRow, .CopyToEnd = CopyToEnd, -- 2.41.0 [application/octet-stream] v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch (7.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/8-v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch) download | inline diff: From f48e7b629a8d15fc70cd4cc4737dd2ad61910cc9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 11:07:14 +0900 Subject: [PATCH v8 06/10] Add support for adding custom COPY FROM format We use the same approach as we used for custom COPY TO format. Now, custom COPY format handler can return COPY TO format routines or COPY FROM format routines based on the "is_from" argument: copy_handler(true) returns CopyToRoutine copy_handler(false) returns CopyFromRoutine --- src/backend/commands/copy.c | 53 +++++++++++++------ src/include/commands/copyapi.h | 2 + .../expected/test_copy_format.out | 12 +++++ .../test_copy_format/sql/test_copy_format.sql | 6 +++ .../test_copy_format/test_copy_format.c | 50 +++++++++++++++-- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index ec6dfff8ab..479f36868c 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -472,12 +472,9 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, } /* custom format */ - if (!is_from) - { - funcargtypes[0] = INTERNALOID; - handlerOid = LookupFuncName(list_make1(makeString(format)), 1, - funcargtypes, true); - } + funcargtypes[0] = INTERNALOID; + handlerOid = LookupFuncName(list_make1(makeString(format)), 1, + funcargtypes, true); if (!OidIsValid(handlerOid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -486,14 +483,36 @@ ProcessCopyOptionCustomFormat(ParseState *pstate, datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from)); routine = DatumGetPointer(datum); - if (routine == NULL || !IsA(routine, CopyToRoutine)) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct", - format, handlerOid), - parser_errposition(pstate, defel->location))); - - opts_out->to_routine = routine; + if (is_from) + { + if (routine == NULL || !IsA(routine, CopyFromRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%s(%u) did not return a " + "CopyFromRoutine struct", + format, handlerOid), + parser_errposition( + pstate, defel->location))); + opts_out->from_routine = routine; + } + else + { + if (routine == NULL || !IsA(routine, CopyToRoutine)) + ereport( + ERROR, + (errcode( + ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY handler function " + "%s(%u) did not return a " + "CopyToRoutine struct", + format, handlerOid), + parser_errposition( + pstate, defel->location))); + opts_out->to_routine = routine; + } } /* @@ -692,7 +711,11 @@ ProcessCopyOptions(ParseState *pstate, { bool processed = false; - if (!is_from) + if (is_from) + processed = + opts_out->from_routine->CopyFromProcessOption( + cstate, defel); + else processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); if (!processed) ereport(ERROR, diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 323e4705d2..ef1bb201c2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -30,6 +30,8 @@ typedef void (*CopyFromEnd_function) (CopyFromState cstate); /* Routines for a COPY FROM format implementation. */ typedef struct CopyFromRoutine { + NodeTag type; + /* * Called for processing one COPY FROM option. This will return false when * the given option is invalid. diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out index 3a24ae7b97..6af69f0eb7 100644 --- a/src/test/modules/test_copy_format/expected/test_copy_format.out +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -1,6 +1,18 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a INT, b INT, c INT); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test FROM stdin WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromProcessOption: "option_before"="before" +NOTICE: CopyFromProcessOption: "option_after"="after" +NOTICE: CopyFromGetFormat +NOTICE: CopyFromStart: natts=3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromEnd COPY public.test TO stdout WITH ( option_before 'before', format 'test_copy_format', diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql index 0eb7ed2e11..94d3c789a0 100644 --- a/src/test/modules/test_copy_format/sql/test_copy_format.sql +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -1,6 +1,12 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a INT, b INT, c INT); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +COPY public.test FROM stdin WITH ( + option_before 'before', + format 'test_copy_format', + option_after 'after' +); +\. COPY public.test TO stdout WITH ( option_before 'before', format 'test_copy_format', diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index a2219afcde..5e1b40e881 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -18,6 +18,50 @@ PG_MODULE_MAGIC; +static bool +CopyFromProcessOption(CopyFromState cstate, DefElem *defel) +{ + ereport(NOTICE, + (errmsg("CopyFromProcessOption: \"%s\"=\"%s\"", + defel->defname, defGetString(defel)))); + return true; +} + +static int16 +CopyFromGetFormat(CopyFromState cstate) +{ + ereport(NOTICE, (errmsg("CopyFromGetFormat"))); + return 0; +} + +static void +CopyFromStart(CopyFromState cstate, TupleDesc tupDesc) +{ + ereport(NOTICE, (errmsg("CopyFromStart: natts=%d", tupDesc->natts))); +} + +static bool +CopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) +{ + ereport(NOTICE, (errmsg("CopyFromOneRow"))); + return false; +} + +static void +CopyFromEnd(CopyFromState cstate) +{ + ereport(NOTICE, (errmsg("CopyFromEnd"))); +} + +static const CopyFromRoutine CopyFromRoutineTestCopyFormat = { + .type = T_CopyFromRoutine, + .CopyFromProcessOption = CopyFromProcessOption, + .CopyFromGetFormat = CopyFromGetFormat, + .CopyFromStart = CopyFromStart, + .CopyFromOneRow = CopyFromOneRow, + .CopyFromEnd = CopyFromEnd, +}; + static bool CopyToProcessOption(CopyToState cstate, DefElem *defel) { @@ -71,7 +115,7 @@ test_copy_format(PG_FUNCTION_ARGS) (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false"))); if (is_from) - elog(ERROR, "COPY FROM isn't supported yet"); - - PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); + PG_RETURN_POINTER(&CopyFromRoutineTestCopyFormat); + else + PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat); } -- 2.41.0 [application/octet-stream] v8-0010-introduce-contrib-pg_copy_json.patch (16.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/9-v8-0010-introduce-contrib-pg_copy_json.patch) download | inline diff: From 7dc6c1c798178f31728d048d4d528181626b3695 Mon Sep 17 00:00:00 2001 From: Zhao Junwang <[email protected]> Date: Sat, 27 Jan 2024 13:34:38 +0800 Subject: [PATCH v8 10/10] introduce contrib/pg_copy_json Signed-off-by: Zhao Junwang <[email protected]> --- contrib/Makefile | 1 + contrib/meson.build | 1 + contrib/pg_copy_json/.gitignore | 4 + contrib/pg_copy_json/Makefile | 23 ++ .../pg_copy_json/expected/pg_copy_json.out | 80 +++++++ contrib/pg_copy_json/meson.build | 34 +++ contrib/pg_copy_json/pg_copy_json--1.0.sql | 9 + contrib/pg_copy_json/pg_copy_json.c | 218 ++++++++++++++++++ contrib/pg_copy_json/pg_copy_json.control | 5 + contrib/pg_copy_json/sql/pg_copy_json.sql | 59 +++++ src/backend/utils/adt/json.c | 5 +- src/include/utils/json.h | 2 + 12 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 contrib/pg_copy_json/.gitignore create mode 100644 contrib/pg_copy_json/Makefile create mode 100644 contrib/pg_copy_json/expected/pg_copy_json.out create mode 100644 contrib/pg_copy_json/meson.build create mode 100644 contrib/pg_copy_json/pg_copy_json--1.0.sql create mode 100644 contrib/pg_copy_json/pg_copy_json.c create mode 100644 contrib/pg_copy_json/pg_copy_json.control create mode 100644 contrib/pg_copy_json/sql/pg_copy_json.sql diff --git a/contrib/Makefile b/contrib/Makefile index da4e2316a3..82cc496aa2 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ pageinspect \ passwordcheck \ pg_buffercache \ + pg_copy_json \ pg_freespacemap \ pg_prewarm \ pg_stat_statements \ diff --git a/contrib/meson.build b/contrib/meson.build index c12dc906ca..38933d15d1 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -45,6 +45,7 @@ subdir('oid2name') subdir('pageinspect') subdir('passwordcheck') subdir('pg_buffercache') +subdir('pg_copy_json') subdir('pgcrypto') subdir('pg_freespacemap') subdir('pg_prewarm') diff --git a/contrib/pg_copy_json/.gitignore b/contrib/pg_copy_json/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/pg_copy_json/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_copy_json/Makefile b/contrib/pg_copy_json/Makefile new file mode 100644 index 0000000000..b0a348d618 --- /dev/null +++ b/contrib/pg_copy_json/Makefile @@ -0,0 +1,23 @@ +# contrib/pg_copy_json//Makefile + +MODULE_big = pg_copy_json +OBJS = \ + $(WIN32RES) \ + pg_copy_json.o +PGFILEDESC = "pg_copy_json - COPY TO JSON (JavaScript Object Notation) format" + +EXTENSION = pg_copy_json +DATA = pg_copy_json--1.0.sql + +REGRESS = test_copy_format + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_copy_json +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_copy_json/expected/pg_copy_json.out b/contrib/pg_copy_json/expected/pg_copy_json.out new file mode 100644 index 0000000000..73633c2303 --- /dev/null +++ b/contrib/pg_copy_json/expected/pg_copy_json.out @@ -0,0 +1,80 @@ +-- +-- COPY TO JSON +-- +CREATE EXTENSION pg_copy_json; +-- test copying in JSON format with various styles +-- of embedded line ending characters +create temp table copytest ( + style text, + test text, + filler int); +insert into copytest values('DOS',E'abc\r\ndef',1); +insert into copytest values('Unix',E'abc\ndef',2); +insert into copytest values('Mac',E'abc\rdef',3); +insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); +copy copytest to stdout with (format 'json'); +{"style":"DOS","test":"abc\r\ndef","filler":1} +{"style":"Unix","test":"abc\ndef","filler":2} +{"style":"Mac","test":"abc\rdef","filler":3} +{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +-- pg_copy_json do not support COPY FROM +copy copytest from stdout with (format 'json'); +ERROR: cannot use JSON mode in COPY FROM +-- test copying in JSON format with various styles +-- of embedded escaped characters +create temp table copyjsontest ( + id bigserial, + f1 text, + f2 timestamptz); +insert into copyjsontest + select g.i, + CASE WHEN g.i % 2 = 0 THEN + 'line with '' in it: ' || g.i::text + ELSE + 'line with " in it: ' || g.i::text + END, + 'Mon Feb 10 17:32:01 1997 PST' + from generate_series(1,5) as g(i); +insert into copyjsontest (f1) values +(E'aaa\"bbb'::text), +(E'aaa\\bbb'::text), +(E'aaa\/bbb'::text), +(E'aaa\bbbb'::text), +(E'aaa\fbbb'::text), +(E'aaa\nbbb'::text), +(E'aaa\rbbb'::text), +(E'aaa\tbbb'::text); +copy copyjsontest to stdout with (format 'json'); +{"id":1,"f1":"line with \" in it: 1","f2":"1997-02-10T17:32:01-08:00"} +{"id":2,"f1":"line with ' in it: 2","f2":"1997-02-10T17:32:01-08:00"} +{"id":3,"f1":"line with \" in it: 3","f2":"1997-02-10T17:32:01-08:00"} +{"id":4,"f1":"line with ' in it: 4","f2":"1997-02-10T17:32:01-08:00"} +{"id":5,"f1":"line with \" in it: 5","f2":"1997-02-10T17:32:01-08:00"} +{"id":1,"f1":"aaa\"bbb","f2":null} +{"id":2,"f1":"aaa\\bbb","f2":null} +{"id":3,"f1":"aaa/bbb","f2":null} +{"id":4,"f1":"aaa\bbbb","f2":null} +{"id":5,"f1":"aaa\fbbb","f2":null} +{"id":6,"f1":"aaa\nbbb","f2":null} +{"id":7,"f1":"aaa\rbbb","f2":null} +{"id":8,"f1":"aaa\tbbb","f2":null} +-- test force array +copy copytest to stdout (format 'json', force_array); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format 'json', force_array true); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format 'json', force_array false); +{"style":"DOS","test":"abc\r\ndef","filler":1} +{"style":"Unix","test":"abc\ndef","filler":2} +{"style":"Mac","test":"abc\rdef","filler":3} +{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} diff --git a/contrib/pg_copy_json/meson.build b/contrib/pg_copy_json/meson.build new file mode 100644 index 0000000000..71f9338267 --- /dev/null +++ b/contrib/pg_copy_json/meson.build @@ -0,0 +1,34 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +pg_copy_json_sources = files( + 'pg_copy_json.c', +) + +if host_system == 'windows' + pg_copy_json_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'pg_copy_json', + '--FILEDESC', 'pg_copy_json - COPY TO JSON format',]) +endif + +pg_copy_json = shared_module('pg_copy_json', + pg_copy_json_sources, + kwargs: contrib_mod_args, +) +contrib_targets += pg_copy_json + +install_data( + 'pg_copy_json--1.0.sql', + 'pg_copy_json.control', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'pg_copy_json', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'pg_copy_json', + ], + }, +} diff --git a/contrib/pg_copy_json/pg_copy_json--1.0.sql b/contrib/pg_copy_json/pg_copy_json--1.0.sql new file mode 100644 index 0000000000..d738a1e7e9 --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json--1.0.sql @@ -0,0 +1,9 @@ +/* contrib/pg_copy_json/copy_json--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_copy_json" to load this file. \quit + +CREATE FUNCTION pg_catalog.json(internal) + RETURNS copy_handler + AS 'MODULE_PATHNAME', 'copy_json' + LANGUAGE C; diff --git a/contrib/pg_copy_json/pg_copy_json.c b/contrib/pg_copy_json/pg_copy_json.c new file mode 100644 index 0000000000..cbfdee8e8b --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json.c @@ -0,0 +1,218 @@ +/*-------------------------------------------------------------------------- + * + * pg_copy_json.c + * COPY TO JSON (JavaScript Object Notation) format. + * + * Portions Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/test_copy_format.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/copy.h" +#include "commands/defrem.h" +#include "funcapi.h" +#include "libpq/libpq.h" +#include "libpq/pqformat.h" +#include "utils/json.h" + +PG_MODULE_MAGIC; + +typedef struct +{ + /* + * Force output of square brackets as array decorations at the beginning + * and end of output, with commas between the rows. + */ + bool force_array; + bool force_array_specified; + + /* need delimiter to start next json array element */ + bool json_row_delim_needed; +} CopyJsonData; + +static inline void +InitCopyJsonData(CopyJsonData *p) +{ + Assert(p); + p->force_array = false; + p->force_array_specified = false; + p->json_row_delim_needed = false; +} + +static void +CopyToJsonSendEndOfRow(CopyToState cstate) +{ + switch (cstate->copy_dest) + { + case COPY_DEST_FILE: + /* Default line termination depends on platform */ +#ifndef WIN32 + CopySendChar(cstate, '\n'); +#else + CopySendString(cstate, "\r\n"); +#endif + break; + case COPY_DEST_FRONTEND: + /* The FE/BE protocol uses \n as newline for all platforms */ + CopySendChar(cstate, '\n'); + break; + default: + break; + } + CopyToStateFlush(cstate); +} + +static bool +CopyToJsonProcessOption(CopyToState cstate, DefElem *defel) +{ + CopyJsonData *p; + + if (cstate->opaque == NULL) + { + MemoryContext oldcontext; + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + cstate->opaque = palloc0(sizeof(CopyJsonData)); + MemoryContextSwitchTo(oldcontext); + InitCopyJsonData(cstate->opaque); + } + + p = (CopyJsonData *)cstate->opaque; + + if (strcmp(defel->defname, "force_array") == 0) + { + if (p->force_array_specified) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("CopyToJsonProcessOption: redundant options \"%s\"=\"%s\"", + defel->defname, defGetString(defel))); + p->force_array_specified = true; + p->force_array = defGetBoolean(defel); + + return true; + } + + return false; +} + +static void +CopyToJsonSendCopyBegin(CopyToState cstate) +{ + StringInfoData buf; + int16 format = 0; + + pq_beginmessage(&buf, PqMsg_CopyOutResponse); + pq_sendbyte(&buf, format); /* overall format */ + /* + * JSON mode is always one non-binary column + */ + pq_sendint16(&buf, 1); + pq_sendint16(&buf, 0); + pq_endmessage(&buf); +} + +static void +CopyToJsonStart(CopyToState cstate, TupleDesc tupDesc) +{ + CopyJsonData *p; + + if (cstate->opaque == NULL) + { + MemoryContext oldcontext; + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + cstate->opaque = palloc0(sizeof(CopyJsonData)); + MemoryContextSwitchTo(oldcontext); + InitCopyJsonData(cstate->opaque); + } + + /* No need to alloc cstate->out_functions */ + + p = (CopyJsonData *)cstate->opaque; + + /* If FORCE_ARRAY has been specified send the open bracket. */ + if (p->force_array) + { + CopySendChar(cstate, '['); + CopyToJsonSendEndOfRow(cstate); + } +} + +static void +CopyToJsonOneRow(CopyToState cstate, TupleTableSlot *slot) +{ + Datum rowdata; + StringInfo result; + CopyJsonData *p; + + Assert(cstate->opaque); + p = (CopyJsonData *)cstate->opaque; + + if(!cstate->rel) + { + for (int i = 0; i < slot->tts_tupleDescriptor->natts; i++) + { + /* Flat-copy the attribute array */ + memcpy(TupleDescAttr(slot->tts_tupleDescriptor, i), + TupleDescAttr(cstate->queryDesc->tupDesc, i), + 1 * sizeof(FormData_pg_attribute)); + } + BlessTupleDesc(slot->tts_tupleDescriptor); + } + rowdata = ExecFetchSlotHeapTupleDatum(slot); + result = makeStringInfo(); + composite_to_json(rowdata, result, false); + + if (p->json_row_delim_needed) + CopySendChar(cstate, ','); + else if (p->force_array) + { + /* first row needs no delimiter */ + CopySendChar(cstate, ' '); + p->json_row_delim_needed = true; + } + CopySendData(cstate, result->data, result->len); + CopyToJsonSendEndOfRow(cstate); +} + +static void +CopyToJsonEnd(CopyToState cstate) +{ + CopyJsonData *p; + + Assert(cstate->opaque); + p = (CopyJsonData *)cstate->opaque; + + /* If FORCE_ARRAY has been specified send the close bracket. */ + if (p->force_array) + { + CopySendChar(cstate, ']'); + CopyToJsonSendEndOfRow(cstate); + } +} + +static const CopyToRoutine CopyToRoutineJson = { + .type = T_CopyToRoutine, + .CopyToProcessOption = CopyToJsonProcessOption, + .CopyToSendCopyBegin = CopyToJsonSendCopyBegin, + .CopyToStart = CopyToJsonStart, + .CopyToOneRow = CopyToJsonOneRow, + .CopyToEnd = CopyToJsonEnd, +}; + +PG_FUNCTION_INFO_V1(copy_json); +Datum +copy_json(PG_FUNCTION_ARGS) +{ + bool is_from = PG_GETARG_BOOL(0); + + if (is_from) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use JSON mode in COPY FROM"))); + + PG_RETURN_POINTER(&CopyToRoutineJson); +} diff --git a/contrib/pg_copy_json/pg_copy_json.control b/contrib/pg_copy_json/pg_copy_json.control new file mode 100644 index 0000000000..90b0a74603 --- /dev/null +++ b/contrib/pg_copy_json/pg_copy_json.control @@ -0,0 +1,5 @@ +# pg_copy_json extension +comment = 'COPY TO JSON format' +default_version = '1.0' +module_pathname = '$libdir/pg_copy_json' +relocatable = true diff --git a/contrib/pg_copy_json/sql/pg_copy_json.sql b/contrib/pg_copy_json/sql/pg_copy_json.sql new file mode 100644 index 0000000000..73e7e514ac --- /dev/null +++ b/contrib/pg_copy_json/sql/pg_copy_json.sql @@ -0,0 +1,59 @@ +-- +-- COPY TO JSON +-- + +CREATE EXTENSION pg_copy_json; + +-- test copying in JSON format with various styles +-- of embedded line ending characters + +create temp table copytest ( + style text, + test text, + filler int); + +insert into copytest values('DOS',E'abc\r\ndef',1); +insert into copytest values('Unix',E'abc\ndef',2); +insert into copytest values('Mac',E'abc\rdef',3); +insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); + +copy copytest to stdout with (format 'json'); + +-- pg_copy_json do not support COPY FROM +copy copytest from stdout with (format 'json'); + +-- test copying in JSON format with various styles +-- of embedded escaped characters + +create temp table copyjsontest ( + id bigserial, + f1 text, + f2 timestamptz); + +insert into copyjsontest + select g.i, + CASE WHEN g.i % 2 = 0 THEN + 'line with '' in it: ' || g.i::text + ELSE + 'line with " in it: ' || g.i::text + END, + 'Mon Feb 10 17:32:01 1997 PST' + from generate_series(1,5) as g(i); + +insert into copyjsontest (f1) values +(E'aaa\"bbb'::text), +(E'aaa\\bbb'::text), +(E'aaa\/bbb'::text), +(E'aaa\bbbb'::text), +(E'aaa\fbbb'::text), +(E'aaa\nbbb'::text), +(E'aaa\rbbb'::text), +(E'aaa\tbbb'::text); + +copy copyjsontest to stdout with (format 'json'); + +-- test force array + +copy copytest to stdout (format 'json', force_array); +copy copytest to stdout (format 'json', force_array true); +copy copytest to stdout (format 'json', force_array false); diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c index d719a61f16..fabd4e611e 100644 --- a/src/backend/utils/adt/json.c +++ b/src/backend/utils/adt/json.c @@ -83,8 +83,6 @@ typedef struct JsonAggState JsonUniqueBuilderState unique_check; } JsonAggState; -static void composite_to_json(Datum composite, StringInfo result, - bool use_line_feeds); static void array_dim_to_json(StringInfo result, int dim, int ndims, int *dims, Datum *vals, bool *nulls, int *valcount, JsonTypeCategory tcategory, Oid outfuncoid, @@ -507,8 +505,9 @@ array_to_json_internal(Datum array, StringInfo result, bool use_line_feeds) /* * Turn a composite / record into JSON. + * Exported so COPY TO can use it. */ -static void +void composite_to_json(Datum composite, StringInfo result, bool use_line_feeds) { HeapTupleHeader td; diff --git a/src/include/utils/json.h b/src/include/utils/json.h index 6d7f1b387d..d5631171ad 100644 --- a/src/include/utils/json.h +++ b/src/include/utils/json.h @@ -17,6 +17,8 @@ #include "lib/stringinfo.h" /* functions in json.c */ +extern void composite_to_json(Datum composite, StringInfo result, + bool use_line_feeds); extern void escape_json(StringInfo buf, const char *str); extern char *JsonEncodeDateTime(char *buf, Datum value, Oid typid, const int *tzp); -- 2.41.0 [application/octet-stream] v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch (4.5K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/10-v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch) download | inline diff: From 3e847de1acb2fd6966ef01192204448711ca3d5e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 14:19:08 +0900 Subject: [PATCH v8 08/10] Add support for implementing custom COPY FROM format as extension * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index a78a790060..f8a194635d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b7e8f627bf..22accc83ab 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -314,8 +314,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + /* * Represents the different dest cases we need to worry about at * the bottom level -- 2.41.0 [application/octet-stream] v8-0007-Export-CopyFromStateData.patch (17.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/11-v8-0007-Export-CopyFromStateData.patch) download | inline diff: From 1ed575fda7f196ea411e9e53dd9c0739f160fb78 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei <[email protected]> Date: Wed, 24 Jan 2024 14:16:29 +0900 Subject: [PATCH v8 07/10] Export CopyFromStateData It's for custom COPY FROM format handlers implemented as extension. This just moves codes. This doesn't change codes except CopySource enum values. CopySource enum values changes aren't required but I did like I did for CopyDest enum values. I changed COPY_ prefix to COPY_SOURCE_ prefix. For example, COPY_FILE to COPY_SOURCE_FILE. Note that this change isn't enough to implement a custom COPY FROM format handler as extension. We'll do the followings in a subsequent commit: 1. Add an opaque space for custom COPY FROM format handler 2. Export CopyReadBinaryData() to read the next data --- src/backend/commands/copyfrom.c | 4 +- src/backend/commands/copyfromparse.c | 10 +- src/include/commands/copy.h | 2 - src/include/commands/copyapi.h | 156 ++++++++++++++++++++++- src/include/commands/copyfrom_internal.h | 150 ---------------------- 5 files changed, 162 insertions(+), 160 deletions(-) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index d556ebb5d6..b4ac7cbd2c 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1710,7 +1710,7 @@ BeginCopyFrom(ParseState *pstate, pg_encoding_to_char(GetDatabaseEncoding())))); } - cstate->copy_src = COPY_FILE; /* default */ + cstate->copy_src = COPY_SOURCE_FILE; /* default */ cstate->whereClause = whereClause; @@ -1829,7 +1829,7 @@ BeginCopyFrom(ParseState *pstate, if (data_source_cb) { progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK; - cstate->copy_src = COPY_CALLBACK; + cstate->copy_src = COPY_SOURCE_CALLBACK; cstate->data_source_cb = data_source_cb; } else if (pipe) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 49632f75e4..a78a790060 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -181,7 +181,7 @@ ReceiveCopyBegin(CopyFromState cstate) for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); - cstate->copy_src = COPY_FRONTEND; + cstate->copy_src = COPY_SOURCE_FRONTEND; cstate->fe_msgbuf = makeStringInfo(); /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); @@ -249,7 +249,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) switch (cstate->copy_src) { - case COPY_FILE: + case COPY_SOURCE_FILE: bytesread = fread(databuf, 1, maxread, cstate->copy_file); if (ferror(cstate->copy_file)) ereport(ERROR, @@ -258,7 +258,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) if (bytesread == 0) cstate->raw_reached_eof = true; break; - case COPY_FRONTEND: + case COPY_SOURCE_FRONTEND: while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof) { int avail; @@ -341,7 +341,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread) bytesread += avail; } break; - case COPY_CALLBACK: + case COPY_SOURCE_CALLBACK: bytesread = cstate->data_source_cb(databuf, minread, maxread); break; } @@ -1099,7 +1099,7 @@ CopyReadLine(CopyFromState cstate) * after \. up to the protocol end of copy data. (XXX maybe better * not to treat \. as special?) */ - if (cstate->copy_src == COPY_FRONTEND) + if (cstate->copy_src == COPY_SOURCE_FRONTEND) { int inbytes; diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index df29d42555..cd41d32074 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -20,8 +20,6 @@ #include "parser/parse_node.h" #include "tcop/dest.h" -typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); - extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index ef1bb201c2..b7e8f627bf 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -14,11 +14,12 @@ #ifndef COPYAPI_H #define COPYAPI_H +#include "commands/trigger.h" #include "executor/execdesc.h" #include "executor/tuptable.h" +#include "nodes/miscnodes.h" #include "nodes/parsenodes.h" -/* This is private in commands/copyfrom.c */ typedef struct CopyFromStateData *CopyFromState; typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel); @@ -162,6 +163,159 @@ typedef struct CopyFormatOptions CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; + +/* + * Represents the different source cases we need to worry about at + * the bottom level + */ +typedef enum CopySource +{ + COPY_SOURCE_FILE, /* from file (or a piped program) */ + COPY_SOURCE_FRONTEND, /* from frontend */ + COPY_SOURCE_CALLBACK, /* from callback function */ +} CopySource; + +/* + * Represents the end-of-line terminator type of the input + */ +typedef enum EolType +{ + EOL_UNKNOWN, + EOL_NL, + EOL_CR, + EOL_CRNL, +} EolType; + +typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); + +/* + * This struct contains all the state variables used throughout a COPY FROM + * operation. + */ +typedef struct CopyFromStateData +{ + /* low-level state data */ + CopySource copy_src; /* type of copy source */ + FILE *copy_file; /* used if copy_src == COPY_FILE */ + StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ + + EolType eol_type; /* EOL type of input */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + Oid conversion_proc; /* encoding conversion function */ + + /* parameters from the COPY command */ + Relation rel; /* relation to copy from */ + List *attnumlist; /* integer list of attnums to copy */ + char *filename; /* filename, or NULL for STDIN */ + bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data */ + + CopyFormatOptions opts; + bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ + Node *whereClause; /* WHERE condition (or NULL) */ + + /* these are just for error messages, see CopyFromErrorCallback */ + const char *cur_relname; /* table name for error messages */ + uint64 cur_lineno; /* line number for error messages */ + const char *cur_attname; /* current att for error messages */ + const char *cur_attval; /* current att value for error messages */ + bool relname_only; /* don't output line number, att, etc. */ + + /* + * Working state + */ + MemoryContext copycontext; /* per-copy execution context */ + + AttrNumber num_defaults; /* count of att that are missing and have + * default value */ + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapper during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ + int *defmap; /* array of default att numbers related to + * missing att */ + ExprState **defexprs; /* array of default att expressions for all + * att */ + bool *defaults; /* if DEFAULT marker was found for + * corresponding att */ + bool volatile_defexprs; /* is any of defexprs volatile? */ + List *range_table; /* single element list of RangeTblEntry */ + List *rteperminfos; /* single element list of RTEPermissionInfo */ + ExprState *qualexpr; + + TransitionCaptureState *transition_capture; + + /* + * These variables are used to reduce overhead in COPY FROM. + * + * attribute_buf holds the separated, de-escaped text for each field of + * the current line. The CopyReadAttributes functions return arrays of + * pointers into this buffer. We avoid palloc/pfree overhead by re-using + * the buffer on each cycle. + * + * In binary COPY FROM, attribute_buf holds the binary data for the + * current field, but the usage is otherwise similar. + */ + StringInfoData attribute_buf; + + /* field raw data pointers found by COPY FROM */ + + int max_fields; + char **raw_fields; + + /* + * Similarly, line_buf holds the whole input line being processed. The + * input cycle is first to read the whole line into line_buf, and then + * extract the individual attribute fields into attribute_buf. line_buf + * is preserved unmodified so that we can display it in error messages if + * appropriate. (In binary mode, line_buf is not used.) + */ + StringInfoData line_buf; + bool line_buf_valid; /* contains the row being processed? */ + + /* + * input_buf holds input data, already converted to database encoding. + * + * In text mode, CopyReadLine parses this data sufficiently to locate line + * boundaries, then transfers the data to line_buf. We guarantee that + * there is a \0 at input_buf[input_buf_len] at all times. (In binary + * mode, input_buf is not used.) + * + * If encoding conversion is not required, input_buf is not a separate + * buffer but points directly to raw_buf. In that case, input_buf_len + * tracks the number of bytes that have been verified as valid in the + * database encoding, and raw_buf_len is the total number of bytes stored + * in the buffer. + */ +#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ + char *input_buf; + int input_buf_index; /* next byte to process */ + int input_buf_len; /* total # of bytes stored */ + bool input_reached_eof; /* true if we reached EOF */ + bool input_reached_error; /* true if a conversion error happened */ + /* Shorthand for number of unconsumed bytes available in input_buf */ +#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) + + /* + * raw_buf holds raw input data read from the data source (file or client + * connection), not yet converted to the database encoding. Like with + * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. + */ +#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ + char *raw_buf; + int raw_buf_index; /* next byte to process */ + int raw_buf_len; /* total # of bytes stored */ + bool raw_reached_eof; /* true if we reached EOF */ + + /* Shorthand for number of unconsumed bytes available in raw_buf */ +#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) + + uint64 bytes_processed; /* number of bytes processed so far */ +} CopyFromStateData; + /* * Represents the different dest cases we need to worry about at * the bottom level diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 921c1513f7..f8f6120255 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -18,28 +18,6 @@ #include "commands/trigger.h" #include "nodes/miscnodes.h" -/* - * Represents the different source cases we need to worry about at - * the bottom level - */ -typedef enum CopySource -{ - COPY_FILE, /* from file (or a piped program) */ - COPY_FRONTEND, /* from frontend */ - COPY_CALLBACK, /* from callback function */ -} CopySource; - -/* - * Represents the end-of-line terminator type of the input - */ -typedef enum EolType -{ - EOL_UNKNOWN, - EOL_NL, - EOL_CR, - EOL_CRNL, -} EolType; - /* * Represents the insert method to be used during COPY FROM. */ @@ -52,134 +30,6 @@ typedef enum CopyInsertMethod * ExecForeignBatchInsert only if valid */ } CopyInsertMethod; -/* - * This struct contains all the state variables used throughout a COPY FROM - * operation. - */ -typedef struct CopyFromStateData -{ - /* low-level state data */ - CopySource copy_src; /* type of copy source */ - FILE *copy_file; /* used if copy_src == COPY_FILE */ - StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */ - - EolType eol_type; /* EOL type of input */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - Oid conversion_proc; /* encoding conversion function */ - - /* parameters from the COPY command */ - Relation rel; /* relation to copy from */ - List *attnumlist; /* integer list of attnums to copy */ - char *filename; /* filename, or NULL for STDIN */ - bool is_program; /* is 'filename' a program to popen? */ - copy_data_source_cb data_source_cb; /* function for reading data */ - - CopyFormatOptions opts; - bool *convert_select_flags; /* per-column CSV/TEXT CS flags */ - Node *whereClause; /* WHERE condition (or NULL) */ - - /* these are just for error messages, see CopyFromErrorCallback */ - const char *cur_relname; /* table name for error messages */ - uint64 cur_lineno; /* line number for error messages */ - const char *cur_attname; /* current att for error messages */ - const char *cur_attval; /* current att value for error messages */ - bool relname_only; /* don't output line number, att, etc. */ - - /* - * Working state - */ - MemoryContext copycontext; /* per-copy execution context */ - - AttrNumber num_defaults; /* count of att that are missing and have - * default value */ - FmgrInfo *in_functions; /* array of input functions for each attrs */ - Oid *typioparams; /* array of element types for in_functions */ - ErrorSaveContext *escontext; /* soft error trapper during in_functions - * execution */ - uint64 num_errors; /* total number of rows which contained soft - * errors */ - int *defmap; /* array of default att numbers related to - * missing att */ - ExprState **defexprs; /* array of default att expressions for all - * att */ - bool *defaults; /* if DEFAULT marker was found for - * corresponding att */ - bool volatile_defexprs; /* is any of defexprs volatile? */ - List *range_table; /* single element list of RangeTblEntry */ - List *rteperminfos; /* single element list of RTEPermissionInfo */ - ExprState *qualexpr; - - TransitionCaptureState *transition_capture; - - /* - * These variables are used to reduce overhead in COPY FROM. - * - * attribute_buf holds the separated, de-escaped text for each field of - * the current line. The CopyReadAttributes functions return arrays of - * pointers into this buffer. We avoid palloc/pfree overhead by re-using - * the buffer on each cycle. - * - * In binary COPY FROM, attribute_buf holds the binary data for the - * current field, but the usage is otherwise similar. - */ - StringInfoData attribute_buf; - - /* field raw data pointers found by COPY FROM */ - - int max_fields; - char **raw_fields; - - /* - * Similarly, line_buf holds the whole input line being processed. The - * input cycle is first to read the whole line into line_buf, and then - * extract the individual attribute fields into attribute_buf. line_buf - * is preserved unmodified so that we can display it in error messages if - * appropriate. (In binary mode, line_buf is not used.) - */ - StringInfoData line_buf; - bool line_buf_valid; /* contains the row being processed? */ - - /* - * input_buf holds input data, already converted to database encoding. - * - * In text mode, CopyReadLine parses this data sufficiently to locate line - * boundaries, then transfers the data to line_buf. We guarantee that - * there is a \0 at input_buf[input_buf_len] at all times. (In binary - * mode, input_buf is not used.) - * - * If encoding conversion is not required, input_buf is not a separate - * buffer but points directly to raw_buf. In that case, input_buf_len - * tracks the number of bytes that have been verified as valid in the - * database encoding, and raw_buf_len is the total number of bytes stored - * in the buffer. - */ -#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */ - char *input_buf; - int input_buf_index; /* next byte to process */ - int input_buf_len; /* total # of bytes stored */ - bool input_reached_eof; /* true if we reached EOF */ - bool input_reached_error; /* true if a conversion error happened */ - /* Shorthand for number of unconsumed bytes available in input_buf */ -#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index) - - /* - * raw_buf holds raw input data read from the data source (file or client - * connection), not yet converted to the database encoding. Like with - * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len]. - */ -#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */ - char *raw_buf; - int raw_buf_index; /* next byte to process */ - int raw_buf_len; /* total # of bytes stored */ - bool raw_reached_eof; /* true if we reached EOF */ - - /* Shorthand for number of unconsumed bytes available in raw_buf */ -#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) - - uint64 bytes_processed; /* number of bytes processed so far */ -} CopyFromStateData; - extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); -- 2.41.0 ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 02:41 Masahiko Sawada <[email protected]> parent: Junwang Zhao <[email protected]> 1 sibling, 1 reply; 38+ messages in thread From: Masahiko Sawada @ 2024-01-29 02:41 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > Hi, > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > Junwang Zhao <[email protected]> wrote: > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > single option, and store the options in the opaque field, but it can not > > > check the relation of two options, for example, considering json format, > > > the `header` option can not be handled by these two functions. > > > > > > I want to find a way when the user specifies the header option, customer > > > handler can error out. > > > > Ah, you want to use a built-in option (such as "header") > > value from a custom handler, right? Hmm, it may be better > > that we call CopyToProcessOption()/CopyFromProcessOption() > > for all options including built-in options. > > > Hmm, still I don't think it can handle all cases, since we don't know > the sequence of the options, we need all the options been parsed > before we check the compatibility of the options, or customer > handlers will need complicated logic to resolve that, which might > lead to ugly code :( > Does it make sense to pass only non-builtin options to the custom format callback after parsing and evaluating the builtin options? That is, we parse and evaluate only the builtin options and populate opts_out first, then pass each rest option to the custom format handler callback. The callback can refer to the builtin option values. The callback is expected to return false if the passed option is not supported. If one of the builtin formats is specified and the rest options list has at least one option, we raise "option %s not recognized" error. IOW it's the core's responsibility to ranse the "option %s not recognized" error, which is in order to raise a consistent error message. Also, I think the core should check the redundant options including bultiin and custom options. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:10 Junwang Zhao <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Junwang Zhao @ 2024-01-29 03:10 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > Hi, > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > Junwang Zhao <[email protected]> wrote: > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > single option, and store the options in the opaque field, but it can not > > > > check the relation of two options, for example, considering json format, > > > > the `header` option can not be handled by these two functions. > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > handler can error out. > > > > > > Ah, you want to use a built-in option (such as "header") > > > value from a custom handler, right? Hmm, it may be better > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > for all options including built-in options. > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > the sequence of the options, we need all the options been parsed > > before we check the compatibility of the options, or customer > > handlers will need complicated logic to resolve that, which might > > lead to ugly code :( > > > > Does it make sense to pass only non-builtin options to the custom > format callback after parsing and evaluating the builtin options? That > is, we parse and evaluate only the builtin options and populate > opts_out first, then pass each rest option to the custom format > handler callback. The callback can refer to the builtin option values. Yeah, I think this makes sense. > The callback is expected to return false if the passed option is not > supported. If one of the builtin formats is specified and the rest > options list has at least one option, we raise "option %s not > recognized" error. IOW it's the core's responsibility to ranse the > "option %s not recognized" error, which is in order to raise a > consistent error message. Also, I think the core should check the > redundant options including bultiin and custom options. It would be good that core could check all the redundant options, but where should core do the book-keeping of all the options? I have no idea about this, in my implementation of pg_copy_json extension, I handle redundant options by adding a xxx_specified field for each xxx. > > Regards, > > -- > Masahiko Sawada > Amazon Web Services: https://aws.amazon.com -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:21 Masahiko Sawada <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Masahiko Sawada @ 2024-01-29 03:21 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote: > > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > > > Hi, > > > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > > Junwang Zhao <[email protected]> wrote: > > > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > > single option, and store the options in the opaque field, but it can not > > > > > check the relation of two options, for example, considering json format, > > > > > the `header` option can not be handled by these two functions. > > > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > > handler can error out. > > > > > > > > Ah, you want to use a built-in option (such as "header") > > > > value from a custom handler, right? Hmm, it may be better > > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > > for all options including built-in options. > > > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > > the sequence of the options, we need all the options been parsed > > > before we check the compatibility of the options, or customer > > > handlers will need complicated logic to resolve that, which might > > > lead to ugly code :( > > > > > > > Does it make sense to pass only non-builtin options to the custom > > format callback after parsing and evaluating the builtin options? That > > is, we parse and evaluate only the builtin options and populate > > opts_out first, then pass each rest option to the custom format > > handler callback. The callback can refer to the builtin option values. > > Yeah, I think this makes sense. > > > The callback is expected to return false if the passed option is not > > supported. If one of the builtin formats is specified and the rest > > options list has at least one option, we raise "option %s not > > recognized" error. IOW it's the core's responsibility to ranse the > > "option %s not recognized" error, which is in order to raise a > > consistent error message. Also, I think the core should check the > > redundant options including bultiin and custom options. > > It would be good that core could check all the redundant options, > but where should core do the book-keeping of all the options? I have > no idea about this, in my implementation of pg_copy_json extension, > I handle redundant options by adding a xxx_specified field for each > xxx. What I imagined is that while parsing the all specified options, we evaluate builtin options and we add non-builtin options to another list. Then when parsing a non-builtin option, we check if this option already exists in the list. If there is, we raise the "option %s not recognized" error.". Once we complete checking all options, we pass each option in the list to the callback. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 03:37 Junwang Zhao <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Junwang Zhao @ 2024-01-29 03:37 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 11:22 AM Masahiko Sawada <[email protected]> wrote: > > On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote: > > > > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote: > > > > > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote: > > > > > > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote: > > > > > > > > > > Hi, > > > > > > > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com> > > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800, > > > > > Junwang Zhao <[email protected]> wrote: > > > > > > > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle > > > > > > single option, and store the options in the opaque field, but it can not > > > > > > check the relation of two options, for example, considering json format, > > > > > > the `header` option can not be handled by these two functions. > > > > > > > > > > > > I want to find a way when the user specifies the header option, customer > > > > > > handler can error out. > > > > > > > > > > Ah, you want to use a built-in option (such as "header") > > > > > value from a custom handler, right? Hmm, it may be better > > > > > that we call CopyToProcessOption()/CopyFromProcessOption() > > > > > for all options including built-in options. > > > > > > > > > Hmm, still I don't think it can handle all cases, since we don't know > > > > the sequence of the options, we need all the options been parsed > > > > before we check the compatibility of the options, or customer > > > > handlers will need complicated logic to resolve that, which might > > > > lead to ugly code :( > > > > > > > > > > Does it make sense to pass only non-builtin options to the custom > > > format callback after parsing and evaluating the builtin options? That > > > is, we parse and evaluate only the builtin options and populate > > > opts_out first, then pass each rest option to the custom format > > > handler callback. The callback can refer to the builtin option values. > > > > Yeah, I think this makes sense. > > > > > The callback is expected to return false if the passed option is not > > > supported. If one of the builtin formats is specified and the rest > > > options list has at least one option, we raise "option %s not > > > recognized" error. IOW it's the core's responsibility to ranse the > > > "option %s not recognized" error, which is in order to raise a > > > consistent error message. Also, I think the core should check the > > > redundant options including bultiin and custom options. > > > > It would be good that core could check all the redundant options, > > but where should core do the book-keeping of all the options? I have > > no idea about this, in my implementation of pg_copy_json extension, > > I handle redundant options by adding a xxx_specified field for each > > xxx. > > What I imagined is that while parsing the all specified options, we > evaluate builtin options and we add non-builtin options to another > list. Then when parsing a non-builtin option, we check if this option > already exists in the list. If there is, we raise the "option %s not > recognized" error.". Once we complete checking all options, we pass > each option in the list to the callback. LGTM. > > Regards, > > -- > Masahiko Sawada > Amazon Web Services: https://aws.amazon.com -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 06:03 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-29 06:03 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800, Junwang Zhao <[email protected]> wrote: > I have been working on a *COPY TO JSON* extension since yesterday, > which is based on your V6 patch set, I'd like to give you more input > so you can make better decisions about the implementation(with only > pg-copy-arrow you might not get everything considered). Thanks! > 0009 is some changes made by me, I changed CopyToGetFormat to > CopyToSendCopyBegin because pg_copy_json need to send different bytes > in SendCopyBegin, get the format code along is not enough Oh, I haven't cared about the case. How about the following API instead? static void SendCopyBegin(CopyToState cstate) { StringInfoData buf; pq_beginmessage(&buf, PqMsg_CopyOutResponse); cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf); pq_endmessage(&buf); cstate->copy_dest = COPY_FRONTEND; } static void CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf) { int16 format = 0; pq_sendbyte(&buf, format); /* overall format */ /* * JSON mode is always one non-binary column */ pq_sendint16(&buf, 1); pq_sendint16(&buf, format); } > 00010 is the pg_copy_json extension, I think this should be a good > case which can utilize the *extendable copy format* feature It seems that it's convenient that we have one more callback for initializing CopyToState::opaque. It's called only once when Copy{To,From}Routine is chosen: typedef struct CopyToRoutine { void (*CopyToInit) (CopyToState cstate); ... }; void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options) { ... foreach(option, options) { DefElem *defel = lfirst_node(DefElem, option); if (strcmp(defel->defname, "format") == 0) { ... opts_out->to_routine = &CopyToRoutineXXX; opts_out->to_routine->CopyToInit(cstate); ... } } ... } > maybe we > should delete copy_test_format if we have this extension as an > example? I haven't read the COPY TO format json thread[1] carefully (sorry), but we may add the JSON format as a built-in format. If we do it, copy_test_format is useful to test the extension API. [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m... Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 06:48 Junwang Zhao <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 38+ messages in thread From: Junwang Zhao @ 2024-01-29 06:48 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 2:03 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800, > Junwang Zhao <[email protected]> wrote: > > > I have been working on a *COPY TO JSON* extension since yesterday, > > which is based on your V6 patch set, I'd like to give you more input > > so you can make better decisions about the implementation(with only > > pg-copy-arrow you might not get everything considered). > > Thanks! > > > 0009 is some changes made by me, I changed CopyToGetFormat to > > CopyToSendCopyBegin because pg_copy_json need to send different bytes > > in SendCopyBegin, get the format code along is not enough > > Oh, I haven't cared about the case. > How about the following API instead? > > static void > SendCopyBegin(CopyToState cstate) > { > StringInfoData buf; > > pq_beginmessage(&buf, PqMsg_CopyOutResponse); > cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf); > pq_endmessage(&buf); > cstate->copy_dest = COPY_FRONTEND; > } > > static void > CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf) > { > int16 format = 0; > > pq_sendbyte(&buf, format); /* overall format */ > /* > * JSON mode is always one non-binary column > */ > pq_sendint16(&buf, 1); > pq_sendint16(&buf, format); > } Make sense to me. > > > 00010 is the pg_copy_json extension, I think this should be a good > > case which can utilize the *extendable copy format* feature > > It seems that it's convenient that we have one more callback > for initializing CopyToState::opaque. It's called only once > when Copy{To,From}Routine is chosen: > > typedef struct CopyToRoutine > { > void (*CopyToInit) (CopyToState cstate); > ... > }; I like this, we can alloc private data in this hook. > > void > ProcessCopyOptions(ParseState *pstate, > CopyFormatOptions *opts_out, > bool is_from, > void *cstate, > List *options) > { > ... > foreach(option, options) > { > DefElem *defel = lfirst_node(DefElem, option); > > if (strcmp(defel->defname, "format") == 0) > { > ... > opts_out->to_routine = &CopyToRoutineXXX; > opts_out->to_routine->CopyToInit(cstate); > ... > } > } > ... > } > > > > maybe we > > should delete copy_test_format if we have this extension as an > > example? > > I haven't read the COPY TO format json thread[1] carefully > (sorry), but we may add the JSON format as a built-in > format. If we do it, copy_test_format is useful to test the > extension API. Yeah, maybe, I have no strong opinion here, pg_copy_json is just a toy extension for discussion. > > [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m... > > > Thanks, > -- > kou -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-29 09:45 Sutou Kouhei <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-29 09:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800, Junwang Zhao <[email protected]> wrote: >> > > Does it make sense to pass only non-builtin options to the custom >> > > format callback after parsing and evaluating the builtin options? That >> > > is, we parse and evaluate only the builtin options and populate >> > > opts_out first, then pass each rest option to the custom format >> > > handler callback. The callback can refer to the builtin option values. >> >> What I imagined is that while parsing the all specified options, we >> evaluate builtin options and we add non-builtin options to another >> list. Then when parsing a non-builtin option, we check if this option >> already exists in the list. If there is, we raise the "option %s not >> recognized" error.". Once we complete checking all options, we pass >> each option in the list to the callback. I implemented this idea and the following ideas: 1. Add init callback for initialization 2. Change GetFormat() to FillCopyXXXResponse() because JSON format always use 1 column 3. FROM only: Eliminate more cstate->opts.csv_mode branches (This is for performance.) See the attached v9 patch set for details. Changes since v7: 0001: * Move CopyToProcessOption() calls to the end of ProcessCopyOptions() for easy to option validation * Add CopyToState::CopyToInit() and call it in ProcessCopyOptionFormatTo() * Change CopyToState::CopyToGetFormat() to CopyToState::CopyToFillCopyOutResponse() and use it in SendCopyBegin() 0002: * Move CopyFromProcessOption() calls to the end of ProcessCopyOptions() for easy to option validation * Add CopyFromState::CopyFromInit() and call it in ProcessCopyOptionFormatFrom() * Change CopyFromState::CopyFromGetFormat() to CopyFromState::CopyFromFillCopyOutResponse() and use it in ReceiveCopyBegin() * Rename NextCopyFromRawFields() to NextCopyFromRawFieldsInternal() and pass the read attributes callback explicitly to eliminate more cstate->opts.csv_mode branches Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 02:11 Masahiko Sawada <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Masahiko Sawada @ 2024-01-30 02:11 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Mon, Jan 29, 2024 at 6:45 PM Sutou Kouhei <[email protected]> wrote: > > Hi, > > In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800, > Junwang Zhao <[email protected]> wrote: > > >> > > Does it make sense to pass only non-builtin options to the custom > >> > > format callback after parsing and evaluating the builtin options? That > >> > > is, we parse and evaluate only the builtin options and populate > >> > > opts_out first, then pass each rest option to the custom format > >> > > handler callback. The callback can refer to the builtin option values. > >> > >> What I imagined is that while parsing the all specified options, we > >> evaluate builtin options and we add non-builtin options to another > >> list. Then when parsing a non-builtin option, we check if this option > >> already exists in the list. If there is, we raise the "option %s not > >> recognized" error.". Once we complete checking all options, we pass > >> each option in the list to the callback. > > I implemented this idea and the following ideas: > > 1. Add init callback for initialization > 2. Change GetFormat() to FillCopyXXXResponse() > because JSON format always use 1 column > 3. FROM only: Eliminate more cstate->opts.csv_mode branches > (This is for performance.) > > See the attached v9 patch set for details. Changes since v7: > > 0001: > > * Move CopyToProcessOption() calls to the end of > ProcessCopyOptions() for easy to option validation > * Add CopyToState::CopyToInit() and call it in > ProcessCopyOptionFormatTo() > * Change CopyToState::CopyToGetFormat() to > CopyToState::CopyToFillCopyOutResponse() and use it in > SendCopyBegin() Thank you for updating the patch! Here are comments on 0001 patch: --- + if (!format_specified) + /* Set the default format. */ + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); + I think we can pass "text" in this case instead of NULL. That way, ProcessCopyOptionFormatTo doesn't need to handle NULL case. We need curly brackets for this "if branch" as follows: if (!format_specifed) { /* Set the default format. */ ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); } --- + /* Process not built-in options. */ + foreach(option, unknown_options) + { + DefElem *defel = lfirst_node(DefElem, option); + bool processed = false; + + if (!is_from) + processed = opts_out->to_routine->CopyToProcessOption(cstate, defel); + if (!processed) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("option \"%s\" not recognized", + defel->defname), + parser_errposition(pstate, defel->location))); + } + list_free(unknown_options); I think we can check the duplicated options in the core as we discussed. --- +static void +CopyToTextBasedInit(CopyToState cstate) +{ +} and +static void +CopyToBinaryInit(CopyToState cstate) +{ +} Do we really need separate callbacks for the same behavior? I think we can have a common init function say CopyToBuitinInit() that does nothing. Or we can make the init callback optional. The same is true for process-option callback. --- List *convert_select; /* list of column names (can be NIL) */ + const CopyToRoutine *to_routine; /* callback routines for COPY TO */ } CopyFormatOptions; I think CopyToStateData is a better place to have CopyToRoutine. copy_data_dest_cb is also there. --- - if (strcmp(fmt, "text") == 0) - /* default format */ ; - else if (strcmp(fmt, "csv") == 0) - opts_out->csv_mode = true; - else if (strcmp(fmt, "binary") == 0) - opts_out->binary = true; + + if (is_from) + { + char *fmt = defGetString(defel); + + if (strcmp(fmt, "text") == 0) + /* default format */ ; + else if (strcmp(fmt, "csv") == 0) + { + opts_out->csv_mode = true; + } + else if (strcmp(fmt, "binary") == 0) + { + opts_out->binary = true; + } else - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt\), - parser_errposition(pstate, defel->location))); + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, defel); The 0002 patch replaces the options checks with ProcessCopyOptionFormatFrom(). However, both ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would set format-related options such as opts_out->csv_mode etc, which seems not elegant. IIUC the reason why we process only the "format" option first is to set the callback functions and call the init callback. So I think we don't necessarily need to do both setting callbacks and setting format-related options together. Probably we can do only the callback stuff first and then set format-related options in the original place we used to do? --- +static void +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf) +{ + int16 format = 0; + int natts = list_length(cstate->attnumlist); + int i; + + pq_sendbyte(buf, format); /* overall format */ + pq_sendint16(buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(buf, format); /* per-column formats */ +} This function and CopyToBinaryFillCopyOutResponse() fill three things: overall format, the number of columns, and per-column formats. While this approach is flexible, extensions will have to understand the format of CopyOutResponse message. An alternative is to have one or more callbacks that return these three things. --- + /* Get info about the columns we need to process. */ + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(Fmgr\Info)); + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + Oid out_func_oid; + bool isvarlena; + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); + + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + } Is preparing the out functions an extension's responsibility? I thought the core could prepare them based on the overall format specified by extensions, as long as the overall format matches the actual data format to send. What do you think? --- + /* + * Called when COPY TO via the PostgreSQL protocol is started. This must + * fill buf as a valid CopyOutResponse message: + * + */ + /*-- + * +--------+--------+--------+--------+--------+ +--------+--------+ + * | Format | N attributes | Attr1's format |...| AttrN's format | + * +--------+--------+--------+--------+--------+ +--------+--------+ + * 0: text 0: text 0: text + * 1: binary 1: binary 1: binary + */ I think this kind of diagram could be missed from being updated when we update the CopyOutResponse format. It's better to refer to the documentation instead. Regards, -- Masahiko Sawada Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 05:45 Sutou Kouhei <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-30 05:45 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900, Masahiko Sawada <[email protected]> wrote: > --- > + if (!format_specified) > + /* Set the default format. */ > + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); > + > > I think we can pass "text" in this case instead of NULL. That way, > ProcessCopyOptionFormatTo doesn't need to handle NULL case. Yes, we can do it. But it needs a DefElem allocation. Is it acceptable? > We need curly brackets for this "if branch" as follows: > > if (!format_specifed) > { > /* Set the default format. */ > ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); > } Oh, sorry. I assumed that pgindent adjusts the style too. > --- > + /* Process not built-in options. */ > + foreach(option, unknown_options) > + { > + DefElem *defel = lfirst_node(DefElem, option); > + bool processed = false; > + > + if (!is_from) > + processed = > opts_out->to_routine->CopyToProcessOption(cstate, defel); > + if (!processed) > + ereport(ERROR, > + (errcode(ERRCODE_SYNTAX_ERROR), > + errmsg("option \"%s\" not recognized", > + defel->defname), > + parser_errposition(pstate, > defel->location))); > + } > + list_free(unknown_options); > > I think we can check the duplicated options in the core as we discussed. Oh, sorry. I missed the part. I'll implement it. > --- > +static void > +CopyToTextBasedInit(CopyToState cstate) > +{ > +} > > and > > +static void > +CopyToBinaryInit(CopyToState cstate) > +{ > +} > > Do we really need separate callbacks for the same behavior? I think we > can have a common init function say CopyToBuitinInit() that does > nothing. Or we can make the init callback optional. > > The same is true for process-option callback. OK. I'll make them optional. > --- > List *convert_select; /* list of column names (can be NIL) */ > + const CopyToRoutine *to_routine; /* callback > routines for COPY TO */ > } CopyFormatOptions; > > I think CopyToStateData is a better place to have CopyToRoutine. > copy_data_dest_cb is also there. We can do it but ProcessCopyOptions() accepts NULL CopyToState for file_fdw. Can we create an empty CopyToStateData internally like we did for opts_out in ProcessCopyOptions()? (But it requires exporting CopyToStateData. We'll export it in a later patch but it's not yet in 0001.) > The 0002 patch replaces the options checks with > ProcessCopyOptionFormatFrom(). However, both > ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would > set format-related options such as opts_out->csv_mode etc, which seems > not elegant. IIUC the reason why we process only the "format" option > first is to set the callback functions and call the init callback. So > I think we don't necessarily need to do both setting callbacks and > setting format-related options together. Probably we can do only the > callback stuff first and then set format-related options in the > original place we used to do? If we do it, we need to write the (strcmp(format, "csv") == 0) condition in copyto.c and copy.c. I wanted to avoid it. I think that the duplication (setting opts_out->csv_mode in copyto.c and copyfrom.c) is not a problem. But it's not a strong opinion. If (strcmp(format, "csv") == 0) duplication is better than opts_out->csv_mode = true duplication, I'll do it. BTW, if we want to make the CSV format implementation more modularized, we will remove opts_out->csv_mode, move CSV related options to CopyToCSVProcessOption() and keep CSV related options in its opaque space. For example, opts_out->force_quote exists in COPY TO opaque space but doesn't exist in COPY FROM opaque space because it's not used in COPY FROM. > +static void > +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf) > +{ > + int16 format = 0; > + int natts = list_length(cstate->attnumlist); > + int i; > + > + pq_sendbyte(buf, format); /* overall format */ > + pq_sendint16(buf, natts); > + for (i = 0; i < natts; i++) > + pq_sendint16(buf, format); /* per-column formats */ > +} > > This function and CopyToBinaryFillCopyOutResponse() fill three things: > overall format, the number of columns, and per-column formats. While > this approach is flexible, extensions will have to understand the > format of CopyOutResponse message. An alternative is to have one or > more callbacks that return these three things. Yes, we can choose the approach. I don't have a strong opinion on which approach to choose. > + /* Get info about the columns we need to process. */ > + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * > sizeof(Fmgr\Info)); > + foreach(cur, cstate->attnumlist) > + { > + int attnum = lfirst_int(cur); > + Oid out_func_oid; > + bool isvarlena; > + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); > + > + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); > + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); > + } > > Is preparing the out functions an extension's responsibility? I > thought the core could prepare them based on the overall format > specified by extensions, as long as the overall format matches the > actual data format to send. What do you think? Hmm. I want to keep the preparation as an extension's responsibility. Because it's not needed for all formats. For example, Apache Arrow FORMAT doesn't need it. And JSON FORMAT doesn't need it too because it use composite_to_json(). > + /* > + * Called when COPY TO via the PostgreSQL protocol is > started. This must > + * fill buf as a valid CopyOutResponse message: > + * > + */ > + /*-- > + * +--------+--------+--------+--------+--------+ +--------+--------+ > + * | Format | N attributes | Attr1's format |...| AttrN's format | > + * +--------+--------+--------+--------+--------+ +--------+--------+ > + * 0: text 0: text 0: text > + * 1: binary 1: binary 1: binary > + */ > > I think this kind of diagram could be missed from being updated when > we update the CopyOutResponse format. It's better to refer to the > documentation instead. It makes sense. I couldn't find the documentation when I wrote it but I found it now...: https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY Is there recommended comment style to refer a documentation? "See doc/src/sgml/protocol.sgml for the CopyOutResponse message details" is OK? Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 07:20 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-30 07:20 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jan 30, 2024 at 02:45:31PM +0900, Sutou Kouhei wrote: > In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900, > Masahiko Sawada <[email protected]> wrote: > >> --- >> + if (!format_specified) >> + /* Set the default format. */ >> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); >> + >> >> I think we can pass "text" in this case instead of NULL. That way, >> ProcessCopyOptionFormatTo doesn't need to handle NULL case. > > Yes, we can do it. But it needs a DefElem allocation. Is it > acceptable? I don't think that there is a need for a DelElem at all here? While I am OK with the choice of calling CopyToInit() in the ProcessCopyOption*() routines that exist to keep the set of callbacks local to copyto.c and copyfrom.c, I think that this should not bother about setting opts_out->csv_mode or opts_out->csv_mode but just set the opts_out->{to,from}_routine callbacks. >> +static void >> +CopyToTextBasedInit(CopyToState cstate) >> +{ >> +} >> >> and >> >> +static void >> +CopyToBinaryInit(CopyToState cstate) >> +{ >> +} >> >> Do we really need separate callbacks for the same behavior? I think we >> can have a common init function say CopyToBuitinInit() that does >> nothing. Or we can make the init callback optional. Keeping empty options does not strike as a bad idea, because this forces extension developers to think about this code path rather than just ignore it. Now, all the Init() callbacks are empty for the in-core callbacks, so I think that we should just remove it entirely for now. Let's keep the core patch a maximum simple. It is always possible to build on top of it depending on what people need. It's been mentioned that JSON would want that, but this also proves that we just don't care about that for all the in-core callbacks, as well. I would choose a minimalistic design for now. >> + /* Get info about the columns we need to process. */ >> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * >> sizeof(Fmgr\Info)); >> + foreach(cur, cstate->attnumlist) >> + { >> + int attnum = lfirst_int(cur); >> + Oid out_func_oid; >> + bool isvarlena; >> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); >> + >> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); >> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); >> + } >> >> Is preparing the out functions an extension's responsibility? I >> thought the core could prepare them based on the overall format >> specified by extensions, as long as the overall format matches the >> actual data format to send. What do you think? > > Hmm. I want to keep the preparation as an extension's > responsibility. Because it's not needed for all formats. For > example, Apache Arrow FORMAT doesn't need it. And JSON > FORMAT doesn't need it too because it use > composite_to_json(). I agree that it could be really useful for extensions to be able to force that. We already know that for the in-core formats we've cared about being able to enforce the way data is handled in input and output. > It makes sense. I couldn't find the documentation when I > wrote it but I found it now...: > https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY > > Is there recommended comment style to refer a documentation? > "See doc/src/sgml/protocol.sgml for the CopyOutResponse > message details" is OK? There are a couple of places in the C code where we refer to SGML docs when it comes to specific details, so using a method like that here to avoid a duplication with the docs sounds sensible for me. I would be really tempted to put my hands on this patch to put into shape a minimal set of changes because I'm caring quite a lot about the performance gains reported with the removal of the "if" checks in the per-row callbacks, and that's one goal of this thread quite independent on the extensibility. Sutou-san, would you be OK with that? -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 08:15 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-30 08:15 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 16:20:54 +0900, Michael Paquier <[email protected]> wrote: >>> + if (!format_specified) >>> + /* Set the default format. */ >>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL); >>> + >>> >>> I think we can pass "text" in this case instead of NULL. That way, >>> ProcessCopyOptionFormatTo doesn't need to handle NULL case. >> >> Yes, we can do it. But it needs a DefElem allocation. Is it >> acceptable? > > I don't think that there is a need for a DelElem at all here? We use defel->location for an error message. (We don't need to set location for the default "text" DefElem.) > While I > am OK with the choice of calling CopyToInit() in the > ProcessCopyOption*() routines that exist to keep the set of callbacks > local to copyto.c and copyfrom.c, I think that this should not bother > about setting opts_out->csv_mode or opts_out->csv_mode but just set > the opts_out->{to,from}_routine callbacks. OK. I'll keep opts_out->{csv_mode,binary} in copy.c. > Now, all the Init() callbacks are empty for the > in-core callbacks, so I think that we should just remove it entirely > for now. Let's keep the core patch a maximum simple. It is always > possible to build on top of it depending on what people need. It's > been mentioned that JSON would want that, but this also proves that we > just don't care about that for all the in-core callbacks, as well. I > would choose a minimalistic design for now. OK. Let's remove Init() callbacks from the first patch set. > I would be really tempted to put my hands on this patch to put into > shape a minimal set of changes because I'm caring quite a lot about > the performance gains reported with the removal of the "if" checks in > the per-row callbacks, and that's one goal of this thread quite > independent on the extensibility. Sutou-san, would you be OK with > that? Yes, sure. (We want to focus on the performance gains in the first patch set and then focus on extensibility again, right?) For the purpose, I think that the v7 patch set is more suitable than the v9 patch set. The v7 patch set doesn't include Init() callbacks, custom options validation support or extra Copy{In,Out}Response support. But the v7 patch set misses the removal of the "if" checks in NextCopyFromRawFields() that exists in the v9 patch set. I'm not sure how much performance will improve by this but it may be worth a try. Can I prepare the v10 patch set as "the v7 patch set" + "the removal of the "if" checks in NextCopyFromRawFields()"? (+ reverting opts_out->{csv_mode,binary} changes in ProcessCopyOptions().) Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-30 08:37 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Michael Paquier @ 2024-01-30 08:37 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Tue, Jan 30, 2024 at 05:15:11PM +0900, Sutou Kouhei wrote: > We use defel->location for an error message. (We don't need > to set location for the default "text" DefElem.) Yeah, but you should not need to have this error in the paths that set the callback routines in opts_out if the same validation happens a few lines before, in copy.c. > Yes, sure. > (We want to focus on the performance gains in the first > patch set and then focus on extensibility again, right?) Yep, exactly, the numbers are too good to just ignore. I don't want to hijack the thread, but I am really worried about the complexities this thread is getting into because we are trying to shape the callbacks in the most generic way possible based on *two* use cases. This is going to be a never-ending discussion. I'd rather get some simple basics, and then we can discuss if tweaking the callbacks is really necessary or not. Even after introducing the pg_proc lookups to get custom callbacks. > For the purpose, I think that the v7 patch set is more > suitable than the v9 patch set. The v7 patch set doesn't > include Init() callbacks, custom options validation support > or extra Copy{In,Out}Response support. But the v7 patch set > misses the removal of the "if" checks in > NextCopyFromRawFields() that exists in the v9 patch set. I'm > not sure how much performance will improve by this but it > may be worth a try. Yeah.. The custom options don't seem like an absolute strong requirement for the first shot with the callbacks or even the possibility to retrieve the callbacks from a function call. I mean, you could provide some control with SET commands and a few GUCs, at least, even if that would be strange. Manipulations with a list of DefElems is the intuitive way to have custom options at query level, but we also have to guess the set of callbacks from this list of DefElems coming from the query. You see my point, I am not sure if it would be the best thing to process twice the options, especially when it comes to decide if a DefElem should be valid or not depending on the callbacks used. Or we could use a kind of "special" DefElem where we could store a set of key:value fed to a callback :) > Can I prepare the v10 patch set as "the v7 patch set" + "the > removal of the "if" checks in NextCopyFromRawFields()"? > (+ reverting opts_out->{csv_mode,binary} changes in > ProcessCopyOptions().) Yep, if I got it that would make sense to me. If you can do that, that would help quite a bit. :) -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-31 05:11 Sutou Kouhei <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 38+ messages in thread From: Sutou Kouhei @ 2024-01-31 05:11 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers Hi, In <[email protected]> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 17:37:35 +0900, Michael Paquier <[email protected]> wrote: >> We use defel->location for an error message. (We don't need >> to set location for the default "text" DefElem.) > > Yeah, but you should not need to have this error in the paths that set > the callback routines in opts_out if the same validation happens a few > lines before, in copy.c. Ah, yes. defel->location is used in later patches. For example, it's used when a COPY handler for the specified FORMAT isn't found. > I am really worried about the complexities > this thread is getting into because we are trying to shape the > callbacks in the most generic way possible based on *two* use cases. > This is going to be a never-ending discussion. I'd rather get some > simple basics, and then we can discuss if tweaking the callbacks is > really necessary or not. Even after introducing the pg_proc lookups > to get custom callbacks. I understand your concern. Let's introduce minimal callbacks as the first step. I think that we completed our design discussion for this feature. We can choose minimal callbacks based on the discussion. > The custom options don't seem like an absolute strong > requirement for the first shot with the callbacks or even the > possibility to retrieve the callbacks from a function call. I mean, > you could provide some control with SET commands and a few GUCs, at > least, even if that would be strange. Manipulations with a list of > DefElems is the intuitive way to have custom options at query level, > but we also have to guess the set of callbacks from this list of > DefElems coming from the query. You see my point, I am not sure > if it would be the best thing to process twice the options, especially > when it comes to decide if a DefElem should be valid or not depending > on the callbacks used. Or we could use a kind of "special" DefElem > where we could store a set of key:value fed to a callback :) Interesting. Let's remove custom options support from the initial minimal callbacks. >> Can I prepare the v10 patch set as "the v7 patch set" + "the >> removal of the "if" checks in NextCopyFromRawFields()"? >> (+ reverting opts_out->{csv_mode,binary} changes in >> ProcessCopyOptions().) > > Yep, if I got it that would make sense to me. If you can do that, > that would help quite a bit. :) I've prepared the v10 patch set. Could you try this? Changes since the v7 patch set: 0001: * Remove CopyToProcessOption() callback * Remove CopyToGetFormat() callback * Revert passing CopyToState to ProcessCopyOptions() * Revert moving "opts_out->{csv_mode,binary} = true" to ProcessCopyOptionFormatTo() * Change to receive "const char *format" instead "DefElem *defel" by ProcessCopyOptionFormatTo() 0002: * Remove CopyFromProcessOption() callback * Remove CopyFromGetFormat() callback * Change to receive "const char *format" instead "DefElem *defel" by ProcessCopyOptionFormatFrom() * Remove "if (cstate->opts.csv_mode)" branches from NextCopyFromRawFields() FYI: Here are Copy{From,To}Routine in the v10 patch set. I think that only Copy{From,To}OneRow are minimal callbacks for the performance gain. But can we keep Copy{From,To}Start and Copy{From,To}End for consistency? We can remove a few {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It doesn't depend on the number of COPY target tuples. So they will not affect performance. /* Routines for a COPY FROM format implementation. */ typedef struct CopyFromRoutine { /* * Called when COPY FROM is started. This will initialize something and * receive a header. */ void (*CopyFromStart) (CopyFromState cstate, TupleDesc tupDesc); /* Copy one row. It returns false if no more tuples. */ bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); /* Called when COPY FROM is ended. This will finalize something. */ void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; /* Routines for a COPY TO format implementation. */ typedef struct CopyToRoutine { /* Called when COPY TO is started. This will send a header. */ void (*CopyToStart) (CopyToState cstate, TupleDesc tupDesc); /* Copy one row for COPY TO. */ void (*CopyToOneRow) (CopyToState cstate, TupleTableSlot *slot); /* Called when COPY TO is ended. This will send a trailer. */ void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; Thanks, -- kou ^ permalink raw reply [nested|flat] 38+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations @ 2024-01-31 05:39 Michael Paquier <[email protected]> parent: Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 38+ messages in thread From: Michael Paquier @ 2024-01-31 05:39 UTC (permalink / raw) To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers On Wed, Jan 31, 2024 at 02:11:22PM +0900, Sutou Kouhei wrote: > Ah, yes. defel->location is used in later patches. For > example, it's used when a COPY handler for the specified > FORMAT isn't found. I see. > I've prepared the v10 patch set. Could you try this? Thanks, I'm looking into that now. > FYI: Here are Copy{From,To}Routine in the v10 patch set. I > think that only Copy{From,To}OneRow are minimal callbacks > for the performance gain. But can we keep Copy{From,To}Start > and Copy{From,To}End for consistency? We can remove a few > {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It > doesn't depend on the number of COPY target tuples. So they > will not affect performance. I think I'm OK to keep the start/end callbacks. This makes the code more consistent as a whole, as well. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 38+ messages in thread
end of thread, other threads:[~2024-01-31 05:39 UTC | newest] Thread overview: 38+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2023-02-16 06:06 [PATCH v20 5/5] introduce restore_library Nathan Bossart <[email protected]> 2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]> 2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]> 2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]> 2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]> 2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]> 2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]> 2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox