public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/7] Make Append node async-aware. 58+ messages / 3 participants [nested] [flat]
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v50 8/8] Reserve replication slots specifically for REPACK @ 2026-04-01 17:54 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Álvaro Herrera @ 2026-04-01 17:54 UTC (permalink / raw) This allows REPACK to not interfere with other operations that use replication slots. This eases configurability. --- doc/src/sgml/config.sgml | 16 ++++ doc/src/sgml/ref/repack.sgml | 6 +- src/backend/commands/repack.c | 3 +- src/backend/commands/repack_worker.c | 4 +- src/backend/replication/logical/launcher.c | 2 +- src/backend/replication/logical/slotsync.c | 5 +- src/backend/replication/slot.c | 78 ++++++++++++------- src/backend/replication/slotfuncs.c | 8 +- src/backend/replication/walsender.c | 4 +- src/backend/utils/misc/guc_parameters.dat | 8 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/replication/slot.h | 6 +- 12 files changed, 96 insertions(+), 46 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fdb77df0fdb..a87626a01b6 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4638,6 +4638,22 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-max-repack-replication-slots" xreflabel="max_repack_replication_slots"> + <term><varname>max_repack_replication_slots</varname> (<type>integer</type>) + <indexterm> + <primary><varname>max_repack_replication_slots</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + Specifies the maximum number of replication slots for use of + the <command>REPACK</command> command. The default is 5. + This parameter can only be set at server start. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-max-replication-slots" xreflabel="max_replication_slots"> <term><varname>max_replication_slots</varname> (<type>integer</type>) <indexterm> diff --git a/doc/src/sgml/ref/repack.sgml b/doc/src/sgml/ref/repack.sgml index bec18c44cc8..1424446ba7c 100644 --- a/doc/src/sgml/ref/repack.sgml +++ b/doc/src/sgml/ref/repack.sgml @@ -293,9 +293,9 @@ REPACK [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] USING <listitem> <para> - The <link linkend="guc-max-replication-slots"><varname>max_replication_slots</varname></link> - configuration parameter does not allow for creation of an additional - replication slot. + The <link linkend="guc-max-repack-replication-slots"><varname>max_repack_replication_slots</varname></link> + configuration parameter does not allow for the creation of an + additional replication slot. </para> </listitem> </itemizedlist> diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index d6e446d582d..2fc30008f59 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -3383,7 +3383,8 @@ start_repack_decoding_worker(Oid relid) ereport(ERROR, errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("out of background worker slots"), - errhint("You might need to increase \"%s\".", "max_worker_processes")); + /* FIXME rename to max_repack_processes? */ + errhint("You might need to increase \"%s\".", "max_repack_replication_slots")); decoding_worker->seg = seg; decoding_worker->error_mqh = mqh; diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c index ff34e246469..610592a05b0 100644 --- a/src/backend/commands/repack_worker.c +++ b/src/backend/commands/repack_worker.c @@ -233,8 +233,8 @@ repack_setup_logical_decoding(Oid relid) * RS_TEMPORARY so that the slot gets cleaned up on ERROR. */ snprintf(NameStr(slotname), NAMEDATALEN, "repack_%d", MyProcPid); - ReplicationSlotCreate(NameStr(slotname), true, RS_TEMPORARY, false, false, - false); + ReplicationSlotCreate(NameStr(slotname), true, RS_TEMPORARY, false, true, + false, false); EnsureLogicalDecodingEnabled(); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 09964198550..d83125afd0d 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -1575,7 +1575,7 @@ CreateConflictDetectionSlot(void) errmsg("creating replication conflict detection slot")); ReplicationSlotCreate(CONFLICT_DETECTION_SLOT, false, RS_PERSISTENT, false, - false, false); + false, false, false); init_conflict_slot_xmin(); } diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index e75db69e3f6..1bc7f3f600d 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -425,7 +425,7 @@ get_local_synced_slots(void) LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (int i = 0; i < max_replication_slots; i++) + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; @@ -814,6 +814,7 @@ synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid, */ ReplicationSlotCreate(remote_slot->name, true, RS_TEMPORARY, remote_slot->two_phase, + false, remote_slot->failover, true); @@ -1691,7 +1692,7 @@ update_synced_slots_inactive_since(void) LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (int i = 0; i < max_replication_slots; i++) + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index d553bd5dbff..49fb10df038 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -152,6 +152,9 @@ ReplicationSlot *MyReplicationSlot = NULL; /* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +int max_repack_replication_slots = 5; /* the maximum number of slots + * for REPACK */ +int TotalMaxReplicationSlots = 0; /* sum of both */ /* * Invalidate replication slots that have remained idle longer than this @@ -191,12 +194,14 @@ ReplicationSlotsShmemSize(void) { Size size = 0; - if (max_replication_slots == 0) + TotalMaxReplicationSlots = max_replication_slots + max_repack_replication_slots; + + if (TotalMaxReplicationSlots == 0) return size; size = offsetof(ReplicationSlotCtlData, replication_slots); size = add_size(size, - mul_size(max_replication_slots, sizeof(ReplicationSlot))); + mul_size(TotalMaxReplicationSlots, sizeof(ReplicationSlot))); return size; } @@ -209,7 +214,7 @@ ReplicationSlotsShmemInit(void) { bool found; - if (max_replication_slots == 0) + if (TotalMaxReplicationSlots == 0) return; ReplicationSlotCtl = (ReplicationSlotCtlData *) @@ -223,7 +228,7 @@ ReplicationSlotsShmemInit(void) /* First time through, so initialize */ MemSet(ReplicationSlotCtl, 0, ReplicationSlotsShmemSize()); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i]; @@ -373,6 +378,7 @@ IsSlotForConflictCheck(const char *name) * db_specific: logical decoding is db specific; if the slot is going to * be used for that pass true, otherwise false. * two_phase: If enabled, allows decoding of prepared transactions. + * repack: If true, use a slot from the pool for REPACK. * failover: If enabled, allows the slot to be synced to standbys so * that logical replication can be resumed after failover. * synced: True if the slot is synchronized from the primary server. @@ -380,10 +386,11 @@ IsSlotForConflictCheck(const char *name) void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase, bool failover, bool synced) + bool two_phase, bool repack, bool failover, bool synced) { ReplicationSlot *slot = NULL; - int i; + int startpoint, + endpoint; Assert(MyReplicationSlot == NULL); @@ -432,12 +439,16 @@ ReplicationSlotCreate(const char *name, bool db_specific, LWLockAcquire(ReplicationSlotAllocationLock, LW_EXCLUSIVE); /* - * Check for name collision, and identify an allocatable slot. We need to - * hold ReplicationSlotControlLock in shared mode for this, so that nobody - * else can change the in_use flags while we're looking at them. + * Check for name collision (across the whole array), and identify an + * allocatable slot (in the array slice specific to our current use case: + * either general, or REPACK only). We need to hold + * ReplicationSlotControlLock in shared mode for this, so that nobody else + * can change the in_use flags while we're looking at them. */ LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + startpoint = repack ? max_replication_slots : 0; + endpoint = repack ? TotalMaxReplicationSlots : max_replication_slots; + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; @@ -445,7 +456,9 @@ ReplicationSlotCreate(const char *name, bool db_specific, ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("replication slot \"%s\" already exists", name))); - if (!s->in_use && slot == NULL) + + if (i >= startpoint && i < endpoint && + !s->in_use && slot == NULL) slot = s; } LWLockRelease(ReplicationSlotControlLock); @@ -548,7 +561,7 @@ SearchNamedReplicationSlot(const char *name, bool need_lock) if (need_lock) LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; @@ -576,7 +589,7 @@ int ReplicationSlotIndex(ReplicationSlot *slot) { Assert(slot >= ReplicationSlotCtl->replication_slots && - slot < ReplicationSlotCtl->replication_slots + max_replication_slots); + slot < ReplicationSlotCtl->replication_slots + TotalMaxReplicationSlots); return slot - ReplicationSlotCtl->replication_slots; } @@ -870,7 +883,7 @@ ReplicationSlotCleanup(bool synced_only) restart: found_valid_logicalslot = false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; @@ -1252,7 +1265,7 @@ ReplicationSlotsComputeRequiredXmin(bool already_locked) if (!already_locked) LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; TransactionId effective_xmin; @@ -1307,7 +1320,7 @@ ReplicationSlotsComputeRequiredLSN(void) Assert(ReplicationSlotCtl != NULL); LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; XLogRecPtr restart_lsn; @@ -1374,12 +1387,12 @@ ReplicationSlotsComputeLogicalRestartLSN(void) XLogRecPtr result = InvalidXLogRecPtr; int i; - if (max_replication_slots <= 0) + if (TotalMaxReplicationSlots <= 0) return InvalidXLogRecPtr; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s; XLogRecPtr restart_lsn; @@ -1454,11 +1467,11 @@ ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive) *nslots = *nactive = 0; - if (max_replication_slots <= 0) + if (TotalMaxReplicationSlots <= 0) return false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s; @@ -1515,13 +1528,13 @@ ReplicationSlotsDropDBSlots(Oid dboid) bool found_valid_logicalslot; bool dropped = false; - if (max_replication_slots <= 0) + if (TotalMaxReplicationSlots <= 0) return; restart: found_valid_logicalslot = false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s; char *slotname; @@ -1618,11 +1631,11 @@ CheckLogicalSlotExists(void) { bool found = false; - if (max_replication_slots <= 0) + if (TotalMaxReplicationSlots <= 0) return false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (int i = 0; i < max_replication_slots; i++) + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s; bool invalidated; @@ -1663,7 +1676,11 @@ CheckSlotRequirements(void) * needs the same check. */ - if (max_replication_slots == 0) + /* + * FIXME how to relax this for repack in a way that doesn't mess + * everything up? + */ + if (TotalMaxReplicationSlots == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("replication slots can only be used if \"max_replication_slots\" > 0"))); @@ -2224,7 +2241,7 @@ InvalidateObsoleteReplicationSlots(uint32 possible_causes, Assert(!(possible_causes & RS_INVAL_WAL_REMOVED) || oldestSegno > 0); Assert(possible_causes != RS_INVAL_NONE); - if (max_replication_slots == 0) + if (TotalMaxReplicationSlots == 0) return invalidated; XLogSegNoOffsetToRecPtr(oldestSegno, 0, wal_segment_size, oldestLSN); @@ -2232,7 +2249,7 @@ InvalidateObsoleteReplicationSlots(uint32 possible_causes, restart: found_valid_logicalslot = false; LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (int i = 0; i < max_replication_slots; i++) + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; bool released_lock = false; @@ -2337,7 +2354,7 @@ CheckPointReplicationSlots(bool is_shutdown) */ LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED); - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; char path[MAXPGPATH]; @@ -2438,7 +2455,7 @@ StartupReplicationSlots(void) FreeDir(replication_dir); /* currently no slots exist, we're done. */ - if (max_replication_slots <= 0) + if (TotalMaxReplicationSlots <= 0) return; /* Now that we have recovered all the data, compute replication xmin */ @@ -2868,7 +2885,7 @@ RestoreSlotFromDisk(const char *name) errhint("Change \"wal_level\" to be \"replica\" or higher."))); /* nothing can be active yet, don't lock anything */ - for (i = 0; i < max_replication_slots; i++) + for (i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *slot; @@ -2910,6 +2927,7 @@ RestoreSlotFromDisk(const char *name) break; } + /* XXX might be misleading if the slots previously in use were REPACK. */ if (!restored) ereport(FATAL, (errmsg("too many replication slots active before shutdown"), diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 9f5e4f998fe..cdb4dbd87c9 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -53,7 +53,7 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, temporary ? RS_TEMPORARY : RS_PERSISTENT, false, - false, false); + false, false, false); if (immediately_reserve) { @@ -146,7 +146,7 @@ create_logical_replication_slot(char *name, char *plugin, */ ReplicationSlotCreate(name, true, temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, - failover, false); + false, failover, false); /* * Ensure the logical decoding is enabled before initializing the logical @@ -270,7 +270,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) currlsn = GetXLogWriteRecPtr(); LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); - for (slotno = 0; slotno < max_replication_slots; slotno++) + for (slotno = 0; slotno < TotalMaxReplicationSlots; slotno++) { ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[slotno]; ReplicationSlot slot_contents; @@ -665,7 +665,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) * managed to create the new slot, we advance the new slot's restart_lsn * to the source slot's updated restart_lsn the second time we lock it. */ - for (int i = 0; i < max_replication_slots; i++) + for (int i = 0; i < TotalMaxReplicationSlots; i++) { ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 2bb3f34dc6d..75ef3419a15 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1220,7 +1220,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false, false, false); + false, false, false, false); if (reserve_wal) { @@ -1251,7 +1251,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase, failover, false); + two_phase, false, failover, false); /* * Do options check early so that we can bail before calling the diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index e556b8844d8..8eb251659b0 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2070,6 +2070,14 @@ max => 'MAX_BACKENDS', }, +{ name => 'max_repack_replication_slots', type => 'int', context => 'PGC_POSTMASTER', group => 'REPLICATION_SENDING', + short_desc => 'Sets the maximum number of replication slots for use by REPACK.', + variable => 'max_repack_replication_slots', + boot_val => '5', + min => '0', + max => 'MAX_BACKENDS', +}, + /* see max_wal_senders */ { name => 'max_replication_slots', type => 'int', context => 'PGC_POSTMASTER', group => 'REPLICATION_SENDING', short_desc => 'Sets the maximum number of simultaneously defined replication slots.', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 2c5e98d1d4d..9e9a390a01b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -348,6 +348,8 @@ # (change requires restart) #max_replication_slots = 10 # max number of replication slots # (change requires restart) +#max_repack_replication_slots = 5 # max number of replication slots for REPACK + # (change requires restart) #wal_keep_size = 0 # in megabytes; 0 disables #max_slot_wal_keep_size = -1 # in megabytes; -1 disables #idle_replication_slot_timeout = 0 # in seconds; 0 disables diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 4b4709f6e2c..88953576894 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -324,9 +324,13 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT int max_repack_replication_slots; extern PGDLLIMPORT char *synchronized_standby_slots; extern PGDLLIMPORT int idle_replication_slot_timeout_secs; +/* only for slotfuncs.c, slotsync.c etc */ +extern int TotalMaxReplicationSlots; + /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); extern void ReplicationSlotsShmemInit(void); @@ -334,7 +338,7 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase, bool failover, + bool two_phase, bool repack, bool failover, bool synced); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); -- 2.47.3 --brnevsqjnuzpyok4-- ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH] Limit PL/Perl scalar copies to work_mem @ 2026-07-06 22:13 Andrey Rachitskiy <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Andrey Rachitskiy @ 2026-07-06 22:13 UTC (permalink / raw) When a PL/Perl function returns a very large text value, sv2cstr() copies the entire Perl string into backend memory with no size check. A user with permission to create untrusted PL/Perl functions can return strings far larger than work_mem and risk getting the backend killed by the OOM killer. Reject Perl strings larger than work_mem * 1024 bytes, capped by MaxAllocSize, before copying them through sv2cstr(). This follows the same work_mem-based limit pattern used elsewhere in the backend. Add a plperl regression test that attempts to return a 16MB string with the default 4MB work_mem setting. Author: Andrey Rachitskiy <[email protected]> --- src/pl/plperl/expected/plperl.out | 8 +++++++ src/pl/plperl/plperl.h | 34 ++++++++++++++++++++++++++++++ src/pl/plperl/sql/plperl.sql | 7 ++++++ 3 files changed, 49 insertions(+) diff --git a/src/pl/plperl/expected/plperl.out b/src/pl/plperl/expected/plperl.out index e3d7c88..50c788b 100644 --- a/src/pl/plperl/expected/plperl.out +++ b/src/pl/plperl/expected/plperl.out @@ -792,3 +792,11 @@ SELECT self_modify(42); 126 (1 row) +-- oversized text results are rejected at the PL boundary +CREATE OR REPLACE FUNCTION perl_oversized_text() RETURNS text AS $$ + return 'x' x (16 * 1024 * 1024); +$$ LANGUAGE plperl; +SELECT perl_oversized_text(); +ERROR: Perl value exceeds maximum allowed size (4194304 bytes) +HINT: Increase work_mem or reduce the result size. +CONTEXT: PL/Perl function "perl_oversized_text" diff --git a/src/pl/plperl/plperl.h b/src/pl/plperl/plperl.h index 4c03f9e..1ccce6d 100644 --- a/src/pl/plperl/plperl.h +++ b/src/pl/plperl/plperl.h @@ -17,6 +17,8 @@ /* defines free() by way of system headers, so must be included before perl.h */ #include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "utils/memutils.h" /* * Pull in Perl headers via a wrapper header, to control the scope of @@ -40,6 +42,36 @@ char *plperl_sv_to_literal(SV *, char *); void plperl_util_elog(int level, SV *msg); +/* + * Maximum byte size for a Perl scalar copied through sv2cstr(). + * + * This follows the same work_mem * 1024 pattern used elsewhere in the + * backend (e.g. reorderbuffer.c, nodeHash.c) and is capped by MaxAllocSize. + */ +static inline Size +plperl_max_scalar_bytes(void) +{ + Size limit = (Size) work_mem * (Size) 1024; + + return Min(limit, MaxAllocSize - 1); +} + +/* + * Reject Perl strings that are too large to copy into backend memory. + */ +static inline void +plperl_check_sv_length(STRLEN len) +{ + Size max_len = plperl_max_scalar_bytes(); + + if ((Size) len > max_len) + erereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("Perl value exceeds maximum allowed size (%zu bytes)", + max_len), + errhint("Increase work_mem or reduce the result size."))); +} + /* helper functions */ /* @@ -127,6 +159,8 @@ sv2cstr(SV *sv) else val = SvPVutf8(sv, len); + plperl_check_sv_length(len); + /* * Now convert to database encoding. We use perl's length in the event we * had an embedded null byte to ensure we error out properly. diff --git a/src/pl/plperl/sql/plperl.sql b/src/pl/plperl/sql/plperl.sql index bb0b8ce..0470a2b 100644 --- a/src/pl/plperl/sql/plperl.sql +++ b/src/pl/plperl/sql/plperl.sql @@ -521,3 +521,10 @@ $$ LANGUAGE plperl; SELECT self_modify(42); SELECT self_modify(42); + +-- oversized text results are rejected at the PL boundary +CREATE OR REPLACE FUNCTION perl_oversized_text() RETURNS text AS $$ + return 'x' x (16 * 1024 * 1024); +$$ LANGUAGE plperl; + +SELECT perl_oversized_text(); --MP_/sDTAhto+gkG/PMs8Alw6m8e-- ^ permalink raw reply [nested|flat] 58+ messages in thread
end of thread, other threads:[~2026-07-06 22:13 UTC | newest] Thread overview: 58+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2026-04-01 17:54 [PATCH v50 8/8] Reserve replication slots specifically for REPACK Álvaro Herrera <[email protected]> 2026-07-06 22:13 [PATCH] Limit PL/Perl scalar copies to work_mem Andrey Rachitskiy <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox