From: Andrey Lepikhov <a.lepikhov@postgrespro.ru>
To: Etsuro Fujita <etsuro.fujita@gmail.com>
To: Justin Pryzby <pryzby@telsasoft.com>
Cc: tsunakawa.takay <tsunakawa.takay@fujitsu.com>
Cc: Zhihong Yu <zyu@yugabyte.com>
Cc: Amit Langote <amitlangote09@gmail.com>
Cc: tanghy.fnst@fujitsu.com, Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>
Cc: Pg Hackers <pgsql-hackers@postgresql.org>
Cc: Tomas Vondra <tomas.vondra@enterprisedb.com>
Cc: houzj.fnst@fujitsu.com
Subject: Fast COPY FROM based on batch insert
Date: Fri, 4 Jun 2021 13:26:29 +0500
Message-ID: <bc489202-9855-7550-d64c-ad2d83c24867@postgrespro.ru> (raw)
In-Reply-To: <CAPmGK15rVM+61WHZ3xS3zS6mq7293zCexVMYV4p3xoyTYo8FLg@mail.gmail.com>
References: <TYAPR01MB2990B8451B4C725242E270F7FE979@TYAPR01MB2990.jpnprd01.prod.outlook.com>
<CALNJ-vTdEpeF0nzMhTPYJAhcZVdtwFN+YEkz4tn6p2gDG7j5sA@mail.gmail.com>
<TYAPR01MB299090411AC25A6B56103869FE979@TYAPR01MB2990.jpnprd01.prod.outlook.com>
<20210304203949.GQ29832@telsasoft.com>
<TYAPR01MB29900C08C2C17747612A28B5FE969@TYAPR01MB2990.jpnprd01.prod.outlook.com>
<e3974716-e77b-9006-b1dc-dc00474d09c3@postgrespro.ru>
<TYAPR01MB29909FDCD92F5EEE68E37767FE649@TYAPR01MB2990.jpnprd01.prod.outlook.com>
<CALNJ-vRi4DjjkCmvmXWWtULzY6PSeEAC9vSpXVNHyVoGEQ8HVQ@mail.gmail.com>
<20210323032322.GW4203@telsasoft.com>
<TYAPR01MB2990124D4D859FE8C7E984C8FE649@TYAPR01MB2990.jpnprd01.prod.outlook.com>
<20210409004956.GN6592@telsasoft.com>
<CAPmGK15rVM+61WHZ3xS3zS6mq7293zCexVMYV4p3xoyTYo8FLg@mail.gmail.com>
Hi,
We still have slow 'COPY FROM' operation for foreign tables in current
master.
Now we have a foreign batch insert operation And I tried to rewrite the
patch [1] with this machinery.
The patch (see in attachment) smaller than [1] and no changes required
in FDW API.
Benchmarking
============
I used two data sets: with a number of 1E6 and 1E7 tuples. As a foreign
server emulation I used loopback FDW links.
Test table:
CREATE TABLE test(a int, payload varchar(80));
Execution time of COPY FROM into single foreign table:
version | 1E6 tuples | 1E7 tuples |
master: | 64s | 775s |
Patch [1]: | 5s | 50s |
Current: | 4s | 42s |
Execution time of the COPY operation into a plane table is 0.8s for 1E6
tuples and 8s for 1E7 tuples.
Execution time of COPY FROM into the table partitioned by three foreign
partitions:
version | 1E6 tuples | 1E7 tuples |
master: | 85s | 900s |
Patch [1]: | 10s | 100s |
Current: | 3.5s | 34s |
But the bulk insert execution time in current implementation strongly
depends on MAX_BUFFERED_TUPLES/BYTES value and in my experiments was
reduced to 50s.
[1]
https://www.postgresql.org/message-id/flat/3d0909dc-3691-a576-208a-90986e55489f%40postgrespro.ru
--
regards,
Andrey Lepikhov
Postgres Professional
Attachments:0001-Implementation-of-a-Bulk-COPY-FROM.patch (text/plain, 23.7K) download
From 715406ce4a98df4e0aecdfdf9d9f59cd3a13101e Mon Sep 17 00:00:00 2001
From: "Andrey V. Lepikhov" <a.lepikhov@postgrespro.ru>
Date: Fri, 4 Jun 2021 13:21:43 +0500
Subject: [PATCH] Implementation of a Bulk 'COPY FROM ...' operation into
foreign/distributed table.
---
.../postgres_fdw/expected/postgres_fdw.out | 46 +++-
contrib/postgres_fdw/sql/postgres_fdw.sql | 45 +++
src/backend/commands/copyfrom.c | 259 ++++++++----------
src/backend/executor/execMain.c | 45 +++
src/backend/executor/execPartition.c | 8 +
src/include/commands/copyfrom_internal.h | 10 -
src/include/executor/executor.h | 1 +
src/include/nodes/execnodes.h | 8 +-
8 files changed, 261 insertions(+), 161 deletions(-)
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.outindex f320a7578d..cb2680c6bd 100644--- a/contrib/postgres_fdw/expected/postgres_fdw.out+++ b/contrib/postgres_fdw/expected/postgres_fdw.out@@ -8164,8 +8164,9 @@ copy rem2 from stdin;
copy rem2 from stdin; -- ERROR
ERROR: new row for relation "loc2" violates check constraint "loc2_f1positive"
DETAIL: Failing row contains (-1, xyzzy).
-CONTEXT: remote SQL command: INSERT INTO public.loc2(f1, f2) VALUES ($1, $2)-COPY rem2, line 1: "-1 xyzzy"+CONTEXT: COPY loc2, line 1: "-1 xyzzy"+remote SQL command: COPY public.loc2(f1, f2) FROM STDIN +COPY rem2, line 2
select * from rem2;
f1 | f2
----+-----
@@ -8176,6 +8177,19 @@ select * from rem2;
alter foreign table rem2 drop constraint rem2_f1positive;
alter table loc2 drop constraint loc2_f1positive;
delete from rem2;
+create table foo (a int) partition by list (a);+create table foo1 (like foo);+create foreign table ffoo1 partition of foo for values in (1)+ server loopback options (table_name 'foo1');+create table foo2 (like foo);+create foreign table ffoo2 partition of foo for values in (2)+ server loopback options (table_name 'foo2');+create function print_new_row() returns trigger language plpgsql as $$+ begin raise notice '%', new; return new; end; $$;+create trigger ffoo1_br_trig before insert on ffoo1+ for each row execute function print_new_row();+copy foo from stdin;+NOTICE: (1)
-- Test local triggers
create trigger trig_stmt_before before insert on rem2
for each statement execute procedure trigger_func();
@@ -8284,6 +8298,34 @@ drop trigger rem2_trig_row_before on rem2;
drop trigger rem2_trig_row_after on rem2;
drop trigger loc2_trig_row_before_insert on loc2;
delete from rem2;
+alter table loc2 drop column f1;+alter table loc2 drop column f2;+copy rem2 from stdin;+ERROR: column "f1" of relation "loc2" does not exist+CONTEXT: remote SQL command: COPY public.loc2(f1, f2) FROM STDIN +COPY rem2, line 3+alter table loc2 add column f1 int;+alter table loc2 add column f2 int;+select * from rem2;+ f1 | f2 +----+----+(0 rows)++-- dropped columns locally and on the foreign server+alter table rem2 drop column f1;+alter table rem2 drop column f2;+copy rem2 from stdin;+select * from rem2;+--+(2 rows)++alter table loc2 drop column f1;+alter table loc2 drop column f2;+copy rem2 from stdin;+select * from rem2;+--+(4 rows)+
-- test COPY FROM with foreign table created in the same transaction
create table loc3 (f1 int, f2 text);
begin;
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sqlindex 17dba77d7e..5576328348 100644--- a/contrib/postgres_fdw/sql/postgres_fdw.sql+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql@@ -2258,6 +2258,23 @@ alter table loc2 drop constraint loc2_f1positive;
delete from rem2;
+create table foo (a int) partition by list (a);+create table foo1 (like foo);+create foreign table ffoo1 partition of foo for values in (1)+ server loopback options (table_name 'foo1');+create table foo2 (like foo);+create foreign table ffoo2 partition of foo for values in (2)+ server loopback options (table_name 'foo2');+create function print_new_row() returns trigger language plpgsql as $$+ begin raise notice '%', new; return new; end; $$;+create trigger ffoo1_br_trig before insert on ffoo1+ for each row execute function print_new_row();++copy foo from stdin;+1+2+\.+
-- Test local triggers
create trigger trig_stmt_before before insert on rem2
for each statement execute procedure trigger_func();
@@ -2358,6 +2375,34 @@ drop trigger loc2_trig_row_before_insert on loc2;
delete from rem2;
+alter table loc2 drop column f1;+alter table loc2 drop column f2;+copy rem2 from stdin;+1 foo+2 bar+\.++alter table loc2 add column f1 int;+alter table loc2 add column f2 int;+select * from rem2;++-- dropped columns locally and on the foreign server+alter table rem2 drop column f1;+alter table rem2 drop column f2;+copy rem2 from stdin;+++\.+select * from rem2;++alter table loc2 drop column f1;+alter table loc2 drop column f2;+copy rem2 from stdin;+++\.+select * from rem2;+
-- test COPY FROM with foreign table created in the same transaction
create table loc3 (f1 int, f2 text);
begin;
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.cindex 40a54ad0bd..4f65601bac 100644--- a/src/backend/commands/copyfrom.c+++ b/src/backend/commands/copyfrom.c@@ -317,54 +317,78 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
cstate->line_buf_valid = false;
save_cur_lineno = cstate->cur_lineno;
- /*- * table_multi_insert may leak memory, so switch to short-lived memory- * context before calling it.- */- oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));- table_multi_insert(resultRelInfo->ri_RelationDesc,- slots,- nused,- mycid,- ti_options,- buffer->bistate);- MemoryContextSwitchTo(oldcontext);-- for (i = 0; i < nused; i++)+ if (resultRelInfo->ri_RelationDesc->rd_rel->relkind == RELKIND_FOREIGN_TABLE)+ {+ int sent = 0;++ Assert(resultRelInfo->ri_BatchSize > 1 &&+ resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert != NULL &&+ resultRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize != NULL);++ /* Flush into foreign table or partition */+ do {+ int batch_size = (resultRelInfo->ri_BatchSize < nused - sent) ?+ resultRelInfo->ri_BatchSize : (nused - sent);++ resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert(estate,+ resultRelInfo,+ slots,+ NULL,+ &batch_size);+ sent += batch_size;+ } while (sent < nused);+ }+ else
{
/*
- * If there are any indexes, update them for all the inserted tuples,- * and run AFTER ROW INSERT triggers.+ * table_multi_insert may leak memory, so switch to short-lived memory+ * context before calling it.
*/
- if (resultRelInfo->ri_NumIndices > 0)- {- List *recheckIndexes;-- cstate->cur_lineno = buffer->linenos[i];- recheckIndexes =- ExecInsertIndexTuples(resultRelInfo,- buffer->slots[i], estate, false, false,- NULL, NIL);- ExecARInsertTriggers(estate, resultRelInfo,- slots[i], recheckIndexes,- cstate->transition_capture);- list_free(recheckIndexes);- }+ oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));+ table_multi_insert(resultRelInfo->ri_RelationDesc,+ slots,+ nused,+ mycid,+ ti_options,+ buffer->bistate);+ MemoryContextSwitchTo(oldcontext);- /*- * There's no indexes, but see if we need to run AFTER ROW INSERT- * triggers anyway.- */- else if (resultRelInfo->ri_TrigDesc != NULL &&- (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||- resultRelInfo->ri_TrigDesc->trig_insert_new_table))+ for (i = 0; i < nused; i++)
{
- cstate->cur_lineno = buffer->linenos[i];- ExecARInsertTriggers(estate, resultRelInfo,- slots[i], NIL, cstate->transition_capture);- }+ /*+ * If there are any indexes, update them for all the inserted tuples,+ * and run AFTER ROW INSERT triggers.+ */+ if (resultRelInfo->ri_NumIndices > 0)+ {+ List *recheckIndexes;++ cstate->cur_lineno = buffer->linenos[i];+ recheckIndexes =+ ExecInsertIndexTuples(resultRelInfo,+ buffer->slots[i], estate, false, false,+ NULL, NIL);+ ExecARInsertTriggers(estate, resultRelInfo,+ slots[i], recheckIndexes,+ cstate->transition_capture);+ list_free(recheckIndexes);+ }++ /*+ * There's no indexes, but see if we need to run AFTER ROW INSERT+ * triggers anyway.+ */+ else if (resultRelInfo->ri_TrigDesc != NULL &&+ (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||+ resultRelInfo->ri_TrigDesc->trig_insert_new_table))+ {+ cstate->cur_lineno = buffer->linenos[i];+ ExecARInsertTriggers(estate, resultRelInfo,+ slots[i], NIL, cstate->transition_capture);+ }- ExecClearTuple(slots[i]);+ ExecClearTuple(slots[i]);+ }
}
/* Mark that all slots are free */
@@ -538,13 +562,11 @@ CopyFrom(CopyFromState cstate)
CommandId mycid = GetCurrentCommandId(true);
int ti_options = 0; /* start with default options for insert */
BulkInsertState bistate = NULL;
- CopyInsertMethod insertMethod;
CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */
int64 processed = 0;
int64 excluded = 0;
bool has_before_insert_row_trig;
bool has_instead_insert_row_trig;
- bool leafpart_use_multi_insert = false;
Assert(cstate->rel);
Assert(list_length(cstate->range_table) == 1);
@@ -654,6 +676,33 @@ CopyFrom(CopyFromState cstate)
resultRelInfo = target_resultRelInfo = makeNode(ResultRelInfo);
ExecInitResultRelation(estate, resultRelInfo, 1);
+ Assert(!target_resultRelInfo->ri_usesMultiInsert);++ /*+ * It's generally more efficient to prepare a bunch of tuples for+ * insertion, and insert them in bulk, for example, with one+ * table_multi_insert() call than call table_tuple_insert() separately for+ * every tuple. However, there are a number of reasons why we might not be+ * able to do this. For example, if there any volatile expressions in the+ * table's default values or in the statement's WHERE clause, which may+ * query the table we are inserting into, buffering tuples might produce+ * wrong results. Also, the relation we are trying to insert into itself+ * may not be amenable to buffered inserts.+ *+ * Note: For partitions, this flag is set considering the target table's+ * flag that is being set here and partition's own properties which are+ * checked by calling ExecMultiInsertAllowed(). It does not matter+ * whether partitions have any volatile default expressions as we use the+ * defaults from the target of the COPY command.+ * Also, the COPY command requires a non-zero input list of attributes.+ * Therefore, the length of the attribute list is checked here.+ */+ if (!cstate->volatile_defexprs &&+ list_length(cstate->attnumlist) > 0 &&+ !contain_volatile_functions(cstate->whereClause))+ target_resultRelInfo->ri_usesMultiInsert =+ ExecMultiInsertAllowed(target_resultRelInfo);+
/* Verify the named relation is a valid target for INSERT */
CheckValidResultRel(resultRelInfo, CMD_INSERT);
@@ -676,6 +725,12 @@ CopyFrom(CopyFromState cstate)
resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
resultRelInfo);
+ if (target_resultRelInfo->ri_usesMultiInsert &&+ resultRelInfo->ri_FdwRoutine != NULL &&+ resultRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize != NULL)+ resultRelInfo->ri_BatchSize =+ resultRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize(resultRelInfo);+
/* Prepare to catch AFTER triggers. */
AfterTriggerBeginQuery();
@@ -703,83 +758,9 @@ CopyFrom(CopyFromState cstate)
cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
&mtstate->ps);
- /*- * It's generally more efficient to prepare a bunch of tuples for- * insertion, and insert them in one table_multi_insert() call, than call- * table_tuple_insert() separately for every tuple. However, there are a- * number of reasons why we might not be able to do this. These are- * explained below.- */- if (resultRelInfo->ri_TrigDesc != NULL &&- (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||- resultRelInfo->ri_TrigDesc->trig_insert_instead_row))- {- /*- * Can't support multi-inserts when there are any BEFORE/INSTEAD OF- * triggers on the table. Such triggers might query the table we're- * inserting into and act differently if the tuples that have already- * been processed and prepared for insertion are not there.- */- insertMethod = CIM_SINGLE;- }- else if (proute != NULL && resultRelInfo->ri_TrigDesc != NULL &&- resultRelInfo->ri_TrigDesc->trig_insert_new_table)- {- /*- * For partitioned tables we can't support multi-inserts when there- * are any statement level insert triggers. It might be possible to- * allow partitioned tables with such triggers in the future, but for- * now, CopyMultiInsertInfoFlush expects that any before row insert- * and statement level insert triggers are on the same relation.- */- insertMethod = CIM_SINGLE;- }- else if (resultRelInfo->ri_FdwRoutine != NULL ||- cstate->volatile_defexprs)- {- /*- * Can't support multi-inserts to foreign tables or if there are any- * volatile default expressions in the table. Similarly to the- * trigger case above, such expressions may query the table we're- * inserting into.- *- * Note: It does not matter if any partitions have any volatile- * default expressions as we use the defaults from the target of the- * COPY command.- */- insertMethod = CIM_SINGLE;- }- else if (contain_volatile_functions(cstate->whereClause))- {- /*- * Can't support multi-inserts if there are any volatile function- * expressions in WHERE clause. Similarly to the trigger case above,- * such expressions may query the table we're inserting into.- */- insertMethod = CIM_SINGLE;- }- else- {- /*- * For partitioned tables, we may still be able to perform bulk- * inserts. However, the possibility of this depends on which types- * of triggers exist on the partition. We must disable bulk inserts- * if the partition is a foreign table or it has any before row insert- * or insert instead triggers (same as we checked above for the parent- * table). Since the partition's resultRelInfos are initialized only- * when we actually need to insert the first tuple into them, we must- * have the intermediate insert method of CIM_MULTI_CONDITIONAL to- * flag that we must later determine if we can use bulk-inserts for- * the partition being inserted into.- */- if (proute)- insertMethod = CIM_MULTI_CONDITIONAL;- else- insertMethod = CIM_MULTI;-+ if (resultRelInfo->ri_usesMultiInsert)
CopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, cstate,
estate, mycid, ti_options);
- }
/*
* If not using batch mode (which allocates slots as needed) set up a
@@ -787,7 +768,7 @@ CopyFrom(CopyFromState cstate)
* one, even if we might batch insert, to read the tuple in the root
* partition's form.
*/
- if (insertMethod == CIM_SINGLE || insertMethod == CIM_MULTI_CONDITIONAL)+ if (!resultRelInfo->ri_usesMultiInsert || proute)
{
singleslot = table_slot_create(resultRelInfo->ri_RelationDesc,
&estate->es_tupleTable);
@@ -830,7 +811,7 @@ CopyFrom(CopyFromState cstate)
ResetPerTupleExprContext(estate);
/* select slot to (initially) load row into */
- if (insertMethod == CIM_SINGLE || proute)+ if (!target_resultRelInfo->ri_usesMultiInsert || proute)
{
myslot = singleslot;
Assert(myslot != NULL);
@@ -838,7 +819,6 @@ CopyFrom(CopyFromState cstate)
else
{
Assert(resultRelInfo == target_resultRelInfo);
- Assert(insertMethod == CIM_MULTI);
myslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
resultRelInfo);
@@ -905,24 +885,14 @@ CopyFrom(CopyFromState cstate)
has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
- /*- * Disable multi-inserts when the partition has BEFORE/INSTEAD- * OF triggers, or if the partition is a foreign partition.- */- leafpart_use_multi_insert = insertMethod == CIM_MULTI_CONDITIONAL &&- !has_before_insert_row_trig &&- !has_instead_insert_row_trig &&- resultRelInfo->ri_FdwRoutine == NULL;-
/* Set the multi-insert buffer to use for this partition. */
- if (leafpart_use_multi_insert)+ if (resultRelInfo->ri_usesMultiInsert)
{
if (resultRelInfo->ri_CopyMultiInsertBuffer == NULL)
CopyMultiInsertInfoSetupBuffer(&multiInsertInfo,
resultRelInfo);
}
- else if (insertMethod == CIM_MULTI_CONDITIONAL &&- !CopyMultiInsertInfoIsEmpty(&multiInsertInfo))+ else if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
{
/*
* Flush pending inserts if this partition can't use
@@ -952,7 +922,7 @@ CopyFrom(CopyFromState cstate)
* rowtype.
*/
map = resultRelInfo->ri_RootToPartitionMap;
- if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert)+ if (!resultRelInfo->ri_usesMultiInsert)
{
/* non batch insert */
if (map != NULL)
@@ -971,9 +941,6 @@ CopyFrom(CopyFromState cstate)
*/
TupleTableSlot *batchslot;
- /* no other path available for partitioned table */- Assert(insertMethod == CIM_MULTI_CONDITIONAL);-
batchslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
resultRelInfo);
@@ -1045,7 +1012,7 @@ CopyFrom(CopyFromState cstate)
ExecPartitionCheck(resultRelInfo, myslot, estate, true);
/* Store the slot in the multi-insert buffer, when enabled. */
- if (insertMethod == CIM_MULTI || leafpart_use_multi_insert)+ if (resultRelInfo->ri_usesMultiInsert)
{
/*
* The slot previously might point into the per-tuple
@@ -1124,11 +1091,8 @@ CopyFrom(CopyFromState cstate)
}
/* Flush any remaining buffered tuples */
- if (insertMethod != CIM_SINGLE)- {- if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))- CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);- }+ if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))+ CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);
/* Done, clean up */
error_context_stack = errcallback.previous;
@@ -1149,12 +1113,11 @@ CopyFrom(CopyFromState cstate)
/* Allow the FDW to shut down */
if (target_resultRelInfo->ri_FdwRoutine != NULL &&
target_resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
- target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,- target_resultRelInfo);+ target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,+ target_resultRelInfo);
/* Tear down the multi-insert buffer data */
- if (insertMethod != CIM_SINGLE)- CopyMultiInsertInfoCleanup(&multiInsertInfo);+ CopyMultiInsertInfoCleanup(&multiInsertInfo);
/* Close all the partitioned tables, leaf partitions, and their indices */
if (proute)
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.cindex b3ce4bae53..922e80089c 100644--- a/src/backend/executor/execMain.c+++ b/src/backend/executor/execMain.c@@ -1256,9 +1256,54 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo,
resultRelInfo->ri_PartitionTupleSlot = NULL; /* ditto */
resultRelInfo->ri_ChildToRootMap = NULL;
resultRelInfo->ri_ChildToRootMapValid = false;
+ resultRelInfo->ri_usesMultiInsert = false;
resultRelInfo->ri_CopyMultiInsertBuffer = NULL;
}
+/*+ * ExecMultiInsertAllowed+ * Does this relation allow caller to use multi-insert mode when+ * inserting rows into it?+ */+bool+ExecMultiInsertAllowed(const ResultRelInfo *rri)+{+ /*+ * Can't support multi-inserts when there are any BEFORE/INSTEAD OF+ * triggers on the table. Such triggers might query the table we're+ * inserting into and act differently if the tuples that have already+ * been processed and prepared for insertion are not there.+ */+ if (rri->ri_TrigDesc != NULL &&+ (rri->ri_TrigDesc->trig_insert_before_row ||+ rri->ri_TrigDesc->trig_insert_instead_row))+ return false;++ /*+ * For partitioned tables we can't support multi-inserts when there are+ * any statement level insert triggers. It might be possible to allow+ * partitioned tables with such triggers in the future, but for now,+ * CopyMultiInsertInfoFlush expects that any before row insert and+ * statement level insert triggers are on the same relation.+ */+ if (rri->ri_RelationDesc->rd_rel->relkind == RELKIND_PARTITIONED_TABLE &&+ rri->ri_TrigDesc != NULL &&+ rri->ri_TrigDesc->trig_insert_new_table)+ return false;++ if (rri->ri_FdwRoutine != NULL &&+ (rri->ri_FdwRoutine->ExecForeignBatchInsert == NULL ||+ rri->ri_FdwRoutine->GetForeignModifyBatchSize == NULL))+ /*+ * Foreign tables don't support multi-inserts, unless their FDW+ * provides the necessary bulk insert interface.+ */+ return false;++ /* OK, caller can use multi-insert on this relation. */+ return true;+}+
/*
* ExecGetTriggerResultRel
* Get a ResultRelInfo for a trigger target relation.
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.cindex 606c920b06..d784692bf8 100644--- a/src/backend/executor/execPartition.c+++ b/src/backend/executor/execPartition.c@@ -514,6 +514,14 @@ ExecInitPartitionInfo(ModifyTableState *mtstate, EState *estate,
rootResultRelInfo,
estate->es_instrument);
+ /*+ * If a partition's root parent isn't allowed to use it, neither is the+ * partition.+ */+ if (rootResultRelInfo->ri_usesMultiInsert)+ leaf_part_rri->ri_usesMultiInsert =+ ExecMultiInsertAllowed(leaf_part_rri);+
/*
* Verify result relation is a valid target for an INSERT. An UPDATE of a
* partition-key becomes a DELETE+INSERT operation, so this check is still
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.hindex 4d68d9cceb..598a68a6f1 100644--- a/src/include/commands/copyfrom_internal.h+++ b/src/include/commands/copyfrom_internal.h@@ -39,16 +39,6 @@ typedef enum EolType
EOL_CRNL
} EolType;
-/*- * Represents the heap insert method to be used during COPY FROM.- */-typedef enum CopyInsertMethod-{- CIM_SINGLE, /* use table_tuple_insert or fdw routine */- CIM_MULTI, /* always use table_multi_insert */- CIM_MULTI_CONDITIONAL /* use table_multi_insert only if valid */-} CopyInsertMethod;-
/*
* This struct contains all the state variables used throughout a COPY FROM
* operation.
diff --git a/src/include/executor/executor.h b/src/include/executor/executor.hindex 3dc03c913e..46a79c5ad8 100644--- a/src/include/executor/executor.h+++ b/src/include/executor/executor.h@@ -203,6 +203,7 @@ extern void InitResultRelInfo(ResultRelInfo *resultRelInfo,
Index resultRelationIndex,
ResultRelInfo *partition_root_rri,
int instrument_options);
+extern bool ExecMultiInsertAllowed(const ResultRelInfo *rri);
extern ResultRelInfo *ExecGetTriggerResultRel(EState *estate, Oid relid);
extern void ExecConstraints(ResultRelInfo *resultRelInfo,
TupleTableSlot *slot, EState *estate);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.hindex 7795a69490..58d5df9874 100644--- a/src/include/nodes/execnodes.h+++ b/src/include/nodes/execnodes.h@@ -521,7 +521,13 @@ typedef struct ResultRelInfo
TupleConversionMap *ri_ChildToRootMap;
bool ri_ChildToRootMapValid;
- /* for use by copyfrom.c when performing multi-inserts */+ /*+ * The following fields are currently only relevant to copyfrom.c.+ * True if okay to use multi-insert on this relation+ */+ bool ri_usesMultiInsert;++ /* Buffer allocated to this relation when using multi-insert mode */
struct CopyMultiInsertBuffer *ri_CopyMultiInsertBuffer;
} ResultRelInfo;
--
2.31.1
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: a.lepikhov@postgrespro.ru, etsuro.fujita@gmail.com, pryzby@telsasoft.com, tsunakawa.takay@fujitsu.com, zyu@yugabyte.com, amitlangote09@gmail.com, ashutosh.bapat.oss@gmail.com, tomas.vondra@enterprisedb.com, houzj.fnst@fujitsu.com
Subject: Re: Fast COPY FROM based on batch insert
In-Reply-To: <bc489202-9855-7550-d64c-ad2d83c24867@postgrespro.ru>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox