agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/7] Make Append node async-aware. 45+ messages / 2 participants [nested] [flat]
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 2.9.2 ----Next_Part(Mon_Aug_29_17_08_36_2016_213)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Mon_Aug_29_17_08_36_2016_213)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++------- src/include/nodes/execnodes.h | 2 + 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..1c0d26e 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { - for (;;) + int n_notready = 0; + PlanState *subnode; + int i, n; + + n = node->as_whichplan; + + for (i = 0 ; i < node->as_nplans ; i++) { - PlanState *subnode; TupleTableSlot *result; + if (node->as_async) + { + if (n >= node->as_nplans) + n = 0; + + if (node->as_finished[n]) + { + n++; + continue; + } + } + /* * figure out which subplan we are currently processing */ - subnode = node->appendplans[node->as_whichplan]; + subnode = node->appendplans[n]; /* - * get a tuple from the subplan + * execute the subplan to get a result if it is not ready yet */ - result = ExecProcNode(subnode); + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + /* + * This subnode is not ready yet when asynchrony is not allowed, + * immediately wait for this subnode. + */ + if (!subnode->result_ready) + { + if (node->as_async) + { + n_notready++; + n++; + continue; + } + ExecAsyncWaitForNode(subnode); + } + + Assert(subnode->result_ready); + + result = ExecConsumeResult((PlanState *)subnode); if (!TupIsNull(result)) { /* - * If the subplan gave us something then return it as-is. We do - * NOT make use of the result slot that was set up in - * ExecInitAppend; there's no need for it. + * If the subplan gave us something then return it + * as-is. We do NOT make use of the result slot that was + * set up in ExecInitAppend; there's no need for it. */ + node->as_whichplan = n; ExecReturnTuple(&node->ps, result); return; } + /* Tuple has been exhausted on this subnode */ + if (node->as_async) + { + node->as_finished[n++] = true; + continue; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + if (n >= node->as_nplans - 1) + break; + n++; + } else - node->as_whichplan--; - if (!exec_append_initialize_next(node)) { - ExecReturnTuple(&node->ps, - ExecClearTuple(node->ps.ps_ResultTupleSlot)); - return; + if (n == 0) + break; + n--; } + } - /* Else loop back and try to get a tuple from the new subplan */ + /* + * We are finished if reached here and no subnodes are not-ready + */ + if (n_notready == 0) + { + node->as_whichplan = n; + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); } } @@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9121537..e4f2bb6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1170,6 +1170,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gentbl.sql" CREATE EXTENSION postgres_fdw; CREATE TABLE pl (a int, b int); CREATE TABLE cl1 (LIKE pl) INHERITS (pl); CREATE TABLE cl2 (LIKE pl) INHERITS (pl); CREATE TABLE cl3 (LIKE pl) INHERITS (pl); CREATE TABLE cl4 (LIKE pl) INHERITS (pl); INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a); INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a); INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a); INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a); CREATE TABLE t0 (LIKE pl); INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a); CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres'); CREATE USER MAPPING FOR public SERVER sv0; CREATE USER MAPPING FOR public SERVER sv1; CREATE USER MAPPING FOR public SERVER sv2; CREATE USER MAPPING FOR public SERVER sv3; CREATE USER MAPPING FOR public SERVER sv4; CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4'); CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1'); CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2'); CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3'); CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4'); CREATE TABLE pf0 (LIKE pl); ALTER FOREIGN TABLE ft10 INHERIT pf0; ALTER FOREIGN TABLE ft20 INHERIT pf0; ALTER FOREIGN TABLE ft30 INHERIT pf0; ALTER FOREIGN TABLE ft40 INHERIT pf0; CREATE table pf1 (LIKE pl); ALTER FOREIGN TABLE ft11 INHERIT pf1; ALTER FOREIGN TABLE ft22 INHERIT pf1; ALTER FOREIGN TABLE ft33 INHERIT pf1; ALTER FOREIGN TABLE ft44 INHERIT pf1; ANALYZE; ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="testrun.sh" #! /bin/bash function do_test() { echo $1 for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG } for i in "t0" "pl" "pf0" "pf1"; do do_test $i done ----Next_Part(Wed_Jul_06_16_29_14_2016_599)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Wed_Jul_06_16_29_14_2016_599)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH 7/7] Make Append node async-aware. @ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw) Change append node to be capable to handle asynchronous children properly. As soon as it receives !async_ready from a child, it moves to the next child and if no child is ready, it sleeps until at least one of them become ready. --- src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++-- src/include/nodes/execnodes.h | 2 ++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index e0ce8c6..9a4063a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -58,6 +58,7 @@ #include "postgres.h" #include "executor/execdebug.h" +#include "executor/execAsync.h" #include "executor/nodeAppend.h" static bool exec_append_initialize_next(AppendState *appendstate); @@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { AppendState *appendstate = makeNode(AppendState); PlanState **appendplanstates; + bool *finished; int nplans; int i; ListCell *lc; @@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags) nplans = list_length(node->appendplans); appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *)); - + finished = (bool *) palloc0(nplans * sizeof(bool)); + /* * create new AppendState for our append node */ appendstate->ps.plan = (Plan *) node; appendstate->ps.state = estate; appendstate->appendplans = appendplanstates; + appendstate->as_finished = finished; appendstate->as_nplans = nplans; + appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0); /* * Miscellaneous initialization @@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) void ExecAppend(AppendState *node) { + int stopplan = node->as_whichplan; + for (;;) { PlanState *subnode; @@ -207,7 +214,36 @@ ExecAppend(AppendState *node) /* * get a tuple from the subplan */ - result = ExecProcNode(subnode); + Assert(!node->as_finished[node->as_whichplan]); + + if (!subnode->result_ready) + ExecExecuteNode(subnode); + + if (!subnode->result_ready) + { + if (node->as_async) + { + /* Move to the next living node */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + /* No node is ready yet, return as not-ready */ + if (node->as_whichplan == stopplan) + return; + + /* Try the next node */ + continue; + } + + /* If not async, immediately wait for this subnode */ + ExecAsyncWaitForNode(subnode); + } + + result = ExecConsumeResult((PlanState *) subnode); if (!TupIsNull(result)) { @@ -220,6 +256,31 @@ ExecAppend(AppendState *node) return; } + if (node->as_async) + { + node->as_finished[node->as_whichplan] = true; + stopplan = node->as_whichplan; + + /* Find the next living subnode */ + do + { + node->as_whichplan = + (node->as_whichplan + 1) % node->as_nplans; + } while (node->as_whichplan != stopplan && + node->as_finished[node->as_whichplan]); + + if (node->as_whichplan != stopplan) + { + stopplan = node->as_whichplan; + continue; + } + + /* All subnodes are exhausted. Finish this node. */ + ExecReturnTuple(&node->ps, + ExecClearTuple(node->ps.ps_ResultTupleSlot)); + return; + } + /* * Go on to the "next" subplan in the appropriate direction. If no * more subplans, return the empty slot set up for us by @@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node) { PlanState *subnode = node->appendplans[i]; + node->as_finished[i] = false; + /* * ExecReScan doesn't know about my subplans, so I have to do * changed-parameter signaling myself. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b72decc..b0a86c5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1177,6 +1177,8 @@ typedef struct AppendState { PlanState ps; /* its first field is NodeTag */ PlanState **appendplans; /* array of PlanStates for my inputs */ + bool as_async; /* true to allow async execution */ + bool *as_finished; /* array of the running state of subplans */ int as_nplans; int as_whichplan; } AppendState; -- 1.8.3.1 ----Next_Part(Thu_Jul_21_18_50_07_2016_597)-- Content-Type: text/plain Content-Disposition: inline Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -- Sent via pgsql-hackers mailing list ([email protected]) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers ----Next_Part(Thu_Jul_21_18_50_07_2016_597)---- ^ permalink raw reply [nested|flat] 45+ messages in thread
* [PATCH v8 1/7] Row pattern recognition patch for raw parser. @ 2023-09-25 05:01 Tatsuo Ishii <[email protected]> 0 siblings, 0 replies; 45+ messages in thread From: Tatsuo Ishii @ 2023-09-25 05:01 UTC (permalink / raw) --- src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++-- src/include/nodes/parsenodes.h | 56 ++++++++ src/include/parser/kwlist.h | 8 ++ src/include/parser/parse_node.h | 1 + 4 files changed, 273 insertions(+), 14 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 7d2032885e..74c2069050 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); DefElem *defelt; SortBy *sortby; WindowDef *windef; + RPCommonSyntax *rpcom; + RPSubsetItem *rpsubset; JoinExpr *jexpr; IndexElem *ielem; StatsElem *selem; @@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); MergeWhenClause *mergewhen; struct KeyActions *keyactions; struct KeyAction *keyaction; + RPSkipTo skipto; } %type <node> stmt toplevel_stmt schema_stmt routine_body_stmt @@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); TriggerTransitions TriggerReferencing vacuum_relation_list opt_vacuum_relation_list drop_option_list pub_obj_list - -%type <node> opt_routine_body + row_pattern_measure_list row_pattern_definition_list + opt_row_pattern_subset_clause + row_pattern_subset_list row_pattern_subset_rhs + row_pattern +%type <rpsubset> row_pattern_subset_item +%type <node> opt_routine_body row_pattern_term %type <groupclause> group_clause %type <list> group_by_list %type <node> group_by_item empty_grouping_set rollup_clause cube_clause @@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <range> relation_expr_opt_alias %type <node> tablesample_clause opt_repeatable_clause %type <target> target_el set_target insert_column_item + row_pattern_measure_item row_pattern_definition +%type <skipto> first_or_last %type <str> generic_option_name %type <node> generic_option_arg @@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <list> window_clause window_definition_list opt_partition_clause %type <windef> window_definition over_clause window_specification opt_frame_clause frame_extent frame_bound +%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to +%type <boolean> opt_row_pattern_initial_or_seek +%type <list> opt_row_pattern_measures %type <ival> opt_window_exclusion_clause %type <str> opt_existing_window_name %type <boolean> opt_if_not_exists @@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); json_object_constructor_null_clause_opt json_array_constructor_null_clause_opt - /* * Non-keyword token types. These are hard-wired into the "flex" lexer. * They must be listed first so that their numeric codes do not depend on @@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS - DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC + DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P DOUBLE_P DROP @@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); HANDLER HAVING HEADER_P HOLD HOUR_P IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE - INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P + INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION @@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED - MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD + MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE @@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); ORDER ORDINALITY OTHERS OUT_P OUTER_P OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER - PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD - PLACING PLANS POLICY + PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST + PATTERN_P PERMUTE PLACING PLANS POLICY POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION @@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROUTINE ROUTINES ROW ROWS RULE - SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT + SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT SEQUENCE SEQUENCES + SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P - SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER + SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM @@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); */ %nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */ %nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP +%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P %left Op OPERATOR /* multi-character ops and user-defined operators */ %left '+' '-' %left '*' '/' '%' @@ -15857,7 +15870,8 @@ over_clause: OVER window_specification ; window_specification: '(' opt_existing_window_name opt_partition_clause - opt_sort_clause opt_frame_clause ')' + opt_sort_clause opt_row_pattern_measures opt_frame_clause + opt_row_pattern_common_syntax ')' { WindowDef *n = makeNode(WindowDef); @@ -15865,10 +15879,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause n->refname = $2; n->partitionClause = $3; n->orderClause = $4; + n->rowPatternMeasures = $5; /* copy relevant fields of opt_frame_clause */ - n->frameOptions = $5->frameOptions; - n->startOffset = $5->startOffset; - n->endOffset = $5->endOffset; + n->frameOptions = $6->frameOptions; + n->startOffset = $6->startOffset; + n->endOffset = $6->endOffset; + n->rpCommonSyntax = $7; n->location = @1; $$ = n; } @@ -15892,6 +15908,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; } | /*EMPTY*/ { $$ = NIL; } ; +/* + * ROW PATTERN_P MEASURES + */ +opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + +row_pattern_measure_list: + row_pattern_measure_item + { $$ = list_make1($1); } + | row_pattern_measure_list ',' row_pattern_measure_item + { $$ = lappend($1, $3); } + ; + +row_pattern_measure_item: + a_expr AS ColLabel + { + $$ = makeNode(ResTarget); + $$->name = $3; + $$->indirection = NIL; + $$->val = (Node *) $1; + $$->location = @1; + } + ; + /* * For frame clauses, we return a WindowDef, but only some fields are used: * frameOptions, startOffset, and endOffset. @@ -16051,6 +16092,143 @@ opt_window_exclusion_clause: | /*EMPTY*/ { $$ = 0; } ; +opt_row_pattern_common_syntax: +opt_row_pattern_skip_to opt_row_pattern_initial_or_seek + PATTERN_P '(' row_pattern ')' + opt_row_pattern_subset_clause + DEFINE row_pattern_definition_list + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = $1->rpSkipTo; + n->rpSkipVariable = $1->rpSkipVariable; + n->initial = $2; + n->rpPatterns = $5; + n->rpSubsetClause = $7; + n->rpDefs = $9; + $$ = n; + } + | /*EMPTY*/ { $$ = NULL; } + ; + +opt_row_pattern_skip_to: + AFTER MATCH SKIP TO NEXT ROW + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = ST_NEXT_ROW; + n->rpSkipVariable = NULL; + $$ = n; + } + | AFTER MATCH SKIP PAST LAST_P ROW + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = ST_PAST_LAST_ROW; + n->rpSkipVariable = NULL; + $$ = n; + } + | AFTER MATCH SKIP TO first_or_last ColId + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = $5; + n->rpSkipVariable = $6; + $$ = n; + } +/* + | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = ST_LAST_VARIABLE; + n->rpSkipVariable = $6; + $$ = n; + } + | AFTER MATCH SKIP TO ColId + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + n->rpSkipTo = ST_VARIABLE; + n->rpSkipVariable = $5; + $$ = n; + } +*/ + | /*EMPTY*/ + { + RPCommonSyntax *n = makeNode(RPCommonSyntax); + /* temporary set default to ST_NEXT_ROW */ + n->rpSkipTo = ST_PAST_LAST_ROW; + n->rpSkipVariable = NULL; + $$ = n; + } + ; + +first_or_last: + FIRST_P { $$ = ST_FIRST_VARIABLE; } + | LAST_P { $$ = ST_LAST_VARIABLE; } + ; + +opt_row_pattern_initial_or_seek: + INITIAL { $$ = true; } + | SEEK + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("SEEK is not supported"), + errhint("Use INITIAL."), + parser_errposition(@1))); + } + | /*EMPTY*/ { $$ = true; } + ; + +row_pattern: + row_pattern_term { $$ = list_make1($1); } + | row_pattern row_pattern_term { $$ = lappend($1, $2); } + ; + +row_pattern_term: + ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); } + | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); } + | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); } + | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); } + ; + +opt_row_pattern_subset_clause: + SUBSET row_pattern_subset_list { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + +row_pattern_subset_list: + row_pattern_subset_item { $$ = list_make1($1); } + | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); } + | /*EMPTY*/ { $$ = NIL; } + ; + +row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')' + { + RPSubsetItem *n = makeNode(RPSubsetItem); + n->name = $1; + n->rhsVariable = $4; + $$ = n; + } + ; + +row_pattern_subset_rhs: + ColId { $$ = list_make1(makeStringConst($1, @1)); } + | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); } + | /*EMPTY*/ { $$ = NIL; } + ; + +row_pattern_definition_list: + row_pattern_definition { $$ = list_make1($1); } + | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); } + ; + +row_pattern_definition: + ColId AS a_expr + { + $$ = makeNode(ResTarget); + $$->name = $1; + $$->indirection = NIL; + $$->val = (Node *) $3; + $$->location = @1; + } + ; /* * Supporting nonterminals for expressions. @@ -17146,6 +17324,7 @@ unreserved_keyword: | INDEXES | INHERIT | INHERITS + | INITIAL | INLINE_P | INPUT_P | INSENSITIVE @@ -17173,6 +17352,7 @@ unreserved_keyword: | MATCHED | MATERIALIZED | MAXVALUE + | MEASURES | MERGE | METHOD | MINUTE_P @@ -17215,6 +17395,9 @@ unreserved_keyword: | PARTITION | PASSING | PASSWORD + | PAST + | PATTERN_P + | PERMUTE | PLANS | POLICY | PRECEDING @@ -17265,6 +17448,7 @@ unreserved_keyword: | SEARCH | SECOND_P | SECURITY + | SEEK | SEQUENCE | SEQUENCES | SERIALIZABLE @@ -17290,6 +17474,7 @@ unreserved_keyword: | STRICT_P | STRIP_P | SUBSCRIPTION + | SUBSET | SUPPORT | SYSID | SYSTEM_P @@ -17477,6 +17662,7 @@ reserved_keyword: | CURRENT_USER | DEFAULT | DEFERRABLE + | DEFINE | DESC | DISTINCT | DO @@ -17639,6 +17825,7 @@ bare_label_keyword: | DEFAULTS | DEFERRABLE | DEFERRED + | DEFINE | DEFINER | DELETE_P | DELIMITER @@ -17714,6 +17901,7 @@ bare_label_keyword: | INDEXES | INHERIT | INHERITS + | INITIAL | INITIALLY | INLINE_P | INNER_P @@ -17763,6 +17951,7 @@ bare_label_keyword: | MATCHED | MATERIALIZED | MAXVALUE + | MEASURES | MERGE | METHOD | MINVALUE @@ -17816,6 +18005,9 @@ bare_label_keyword: | PARTITION | PASSING | PASSWORD + | PAST + | PATTERN_P + | PERMUTE | PLACING | PLANS | POLICY @@ -17872,6 +18064,7 @@ bare_label_keyword: | SCROLL | SEARCH | SECURITY + | SEEK | SELECT | SEQUENCE | SEQUENCES @@ -17903,6 +18096,7 @@ bare_label_keyword: | STRICT_P | STRIP_P | SUBSCRIPTION + | SUBSET | SUBSTRING | SUPPORT | SYMMETRIC diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index fef4c714b8..657651df1d 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -547,6 +547,44 @@ typedef struct SortBy int location; /* operator location, or -1 if none/unknown */ } SortBy; +/* + * AFTER MATCH row pattern skip to types in row pattern common syntax + */ +typedef enum RPSkipTo +{ + ST_NONE, /* AFTER MATCH omitted */ + ST_NEXT_ROW, /* SKIP TO NEXT ROW */ + ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */ + ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */ + ST_LAST_VARIABLE, /* SKIP TO LAST variable name */ + ST_VARIABLE /* SKIP TO variable name */ +} RPSkipTo; + +/* + * Row Pattern SUBSET clause item + */ +typedef struct RPSubsetItem +{ + NodeTag type; + char *name; /* Row Pattern SUBSET clause variable name */ + List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */ +} RPSubsetItem; + +/* + * RowPatternCommonSyntax - raw representation of row pattern common syntax + * + */ +typedef struct RPCommonSyntax +{ + NodeTag type; + RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */ + char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */ + bool initial; /* true if <row pattern initial or seek> is initial */ + List *rpPatterns; /* PATTERN variables (list of A_Expr) */ + List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */ + List *rpDefs; /* row pattern definitions clause (list of ResTarget) */ +} RPCommonSyntax; + /* * WindowDef - raw representation of WINDOW and OVER clauses * @@ -562,6 +600,8 @@ typedef struct WindowDef char *refname; /* referenced window name, if any */ List *partitionClause; /* PARTITION BY expression list */ List *orderClause; /* ORDER BY (list of SortBy) */ + List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */ + RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */ int frameOptions; /* frame_clause options, see below */ Node *startOffset; /* expression for starting bound, if any */ Node *endOffset; /* expression for ending bound, if any */ @@ -1483,6 +1523,11 @@ typedef struct GroupingSet * the orderClause might or might not be copied (see copiedOrder); the framing * options are never copied, per spec. * + * "defineClause" is Row Pattern Recognition DEFINE clause (list of + * TargetEntry). TargetEntry.resname represents row pattern definition + * variable name. "patternVariable" and "patternRegexp" represents PATTERN + * clause. + * * The information relevant for the query jumbling is the partition clause * type and its bounds. */ @@ -1514,6 +1559,17 @@ typedef struct WindowClause Index winref; /* ID referenced by window functions */ /* did we copy orderClause from refname? */ bool copiedOrder pg_node_attr(query_jumble_ignore); + /* Row Pattern AFTER MACH SKIP clause */ + RPSkipTo rpSkipTo; /* Row Pattern Skip To type */ + bool initial; /* true if <row pattern initial or seek> is initial */ + /* Row Pattern DEFINE clause (list of TargetEntry) */ + List *defineClause; + /* Row Pattern DEFINE variable initial names (list of String) */ + List *defineInitial; + /* Row Pattern PATTERN variable name (list of String) */ + List *patternVariable; + /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */ + List *patternRegexp; } WindowClause; /* diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 5984dcfa4b..2804333b53 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL) @@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) @@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL) @@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL) @@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL) @@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h index f589112d5e..6640090910 100644 --- a/src/include/parser/parse_node.h +++ b/src/include/parser/parse_node.h @@ -51,6 +51,7 @@ typedef enum ParseExprKind EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */ EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */ EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */ + EXPR_KIND_RPR_DEFINE, /* DEFINE */ EXPR_KIND_SELECT_TARGET, /* SELECT target list item */ EXPR_KIND_INSERT_TARGET, /* INSERT target list item */ EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */ -- 2.25.1 ----Next_Part(Mon_Sep_25_14_26_30_2023_752)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v8-0002-Row-pattern-recognition-patch-parse-analysis.patch" ^ permalink raw reply [nested|flat] 45+ messages in thread
end of thread, other threads:[~2023-09-25 05:01 UTC | newest] Thread overview: 45+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]> 2023-09-25 05:01 [PATCH v8 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox