public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 7/7] Make Append node async-aware.
57+ messages / 2 participants
[nested] [flat]

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH v13 3/8] Row pattern recognition patch (rewriter).
@ 2024-01-22 09:45  Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Tatsuo Ishii @ 2024-01-22 09:45 UTC (permalink / raw)

---
 src/backend/utils/adt/ruleutils.c | 90 +++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 0b2a164057..baded88201 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -428,6 +428,10 @@ static void get_rule_groupingset(GroupingSet *gset, List *targetlist,
 								 bool omit_parens, deparse_context *context);
 static void get_rule_orderby(List *orderList, List *targetList,
 							 bool force_colno, deparse_context *context);
+static void get_rule_pattern(List *patternVariable, List *patternRegexp,
+							 bool force_colno, deparse_context *context);
+static void get_rule_define(List *defineClause, List *patternVariables,
+							bool force_colno, deparse_context *context);
 static void get_rule_windowclause(Query *query, deparse_context *context);
 static void get_rule_windowspec(WindowClause *wc, List *targetList,
 								deparse_context *context);
@@ -6459,6 +6463,64 @@ get_rule_orderby(List *orderList, List *targetList,
 	}
 }
 
+/*
+ * Display a PATTERN clause.
+ */
+static void
+get_rule_pattern(List *patternVariable, List *patternRegexp,
+				 bool force_colno, deparse_context *context)
+{
+	StringInfo  buf = context->buf;
+	const char *sep;
+	ListCell   *lc_var, *lc_reg = list_head(patternRegexp);
+
+	sep = "";
+	appendStringInfoChar(buf, '(');
+	foreach(lc_var, patternVariable)
+	{
+		char *variable = strVal((String *) lfirst(lc_var));
+		char *regexp = NULL;
+
+		if (lc_reg != NULL)
+		{
+			regexp = strVal((String *) lfirst(lc_reg));
+			lc_reg = lnext(patternRegexp, lc_reg);
+		}
+
+		appendStringInfo(buf, "%s%s", sep, variable);
+		if (regexp != NULL)
+			appendStringInfoString(buf, regexp);
+
+		sep = " ";
+	}
+	appendStringInfoChar(buf, ')');
+}
+
+/*
+ * Display a DEFINE clause.
+ */
+static void
+get_rule_define(List *defineClause, List *patternVariables,
+				bool force_colno, deparse_context *context)
+{
+	StringInfo  buf = context->buf;
+	const char *sep;
+	ListCell   *lc_var, *lc_def;
+
+	sep = "  ";
+	Assert(list_length(patternVariables) == list_length(defineClause));
+
+	forboth(lc_var, patternVariables, lc_def, defineClause)
+	{
+		char *varName = strVal(lfirst(lc_var));
+		TargetEntry *te = (TargetEntry *) lfirst(lc_def);
+
+		appendStringInfo(buf, "%s%s AS ", sep, varName);
+		get_rule_expr((Node *) te->expr, context, false);
+		sep = ",\n  ";
+	}
+}
+
 /*
  * Display a WINDOW clause.
  *
@@ -6596,6 +6658,34 @@ get_rule_windowspec(WindowClause *wc, List *targetList,
 			appendStringInfoString(buf, "EXCLUDE GROUP ");
 		else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES)
 			appendStringInfoString(buf, "EXCLUDE TIES ");
+		/* RPR */
+		if (wc->rpSkipTo == ST_NEXT_ROW)
+			appendStringInfoString(buf, "\n  AFTER MATCH SKIP TO NEXT ROW ");
+		else if (wc->rpSkipTo == ST_PAST_LAST_ROW)
+			appendStringInfoString(buf, "\n  AFTER MATCH SKIP PAST LAST ROW ");
+		else if (wc->rpSkipTo == ST_FIRST_VARIABLE)
+			appendStringInfo(buf, "\n  AFTER MATCH SKIP TO FIRST %s ", wc->rpSkipVariable);
+		else if (wc->rpSkipTo == ST_LAST_VARIABLE)
+			appendStringInfo(buf, "\n  AFTER MATCH SKIP TO LAST %s ", wc->rpSkipVariable);
+		else if (wc->rpSkipTo == ST_VARIABLE)
+			appendStringInfo(buf, "\n  AFTER MATCH SKIP TO %s ", wc->rpSkipVariable);
+
+		if (wc->initial)
+			appendStringInfoString(buf, "\n  INITIAL");
+
+		if (wc->patternVariable)
+		{
+			appendStringInfoString(buf, "\n  PATTERN ");
+			get_rule_pattern(wc->patternVariable, wc->patternRegexp, false, context);
+		}
+
+		if (wc->defineClause)
+		{
+			appendStringInfoString(buf, "\n  DEFINE\n");
+			get_rule_define(wc->defineClause, wc->patternVariable, false, context);
+			appendStringInfoChar(buf, ' ');
+		}
+
 		/* we will now have a trailing space; remove it */
 		buf->len--;
 	}
-- 
2.25.1


----Next_Part(Mon_Jan_22_19_26_18_2024_011)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v13-0004-Row-pattern-recognition-patch-planner.patch"



^ permalink  raw  reply  [nested|flat] 57+ messages in thread

end of thread, other threads:[~2024-01-22 09:45 UTC | newest]

Thread overview: 57+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2024-01-22 09:45 [PATCH v13 3/8] Row pattern recognition patch (rewriter). Tatsuo Ishii <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox