public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 7/7] Make Append node async-aware.
57+ messages / 2 participants
[nested] [flat]

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH v44b 1/4] Make index_concurrently_create_copy more general
@ 2026-03-24 18:02 Álvaro Herrera <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Álvaro Herrera @ 2026-03-24 18:02 UTC (permalink / raw)

Add a 'boolean concurrent' option, and make it work for both cases.
Also rename it to index_create_copy.

This allows it to be reused for other purposes -- specifically, for
REPACK CONCURRENTLY.

With the CONCURRENTLY option, REPACK cannot simply swap the heap file and
rebuild its indexes. Instead, it needs to build a separate set of indexes
(including system catalog entries) *before* the actual swap, to reduce the
time AccessExclusiveLock needs to be held for.  This approach is
different from what CREATE INDEX CONCURRENTLY does.

Per a suggestion from Mihail Nikalayeu.

Author: Antonin Houska <[email protected]>
Discussion: https://postgr.es/m/41104.1754922120@localhost
---
 src/backend/catalog/index.c      | 39 +++++++++++++++++++-------------
 src/backend/commands/indexcmds.c | 15 +++++++-----
 src/backend/nodes/makefuncs.c    |  9 ++++----
 src/include/catalog/index.h      |  7 +++---
 src/include/nodes/makefuncs.h    |  4 +++-
 5 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index d8219b18c48..de7182a85a9 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1289,17 +1289,17 @@ index_create(Relation heapRelation,
 }
 
 /*
- * index_concurrently_create_copy
+ * index_create_copy
  *
- * Create concurrently an index based on the definition of the one provided by
- * caller.  The index is inserted into catalogs and needs to be built later
- * on.  This is called during concurrent reindex processing.
+ * Create an index based on the definition of the one provided by caller.  The
+ * index is inserted into catalogs. If 'concurrently' is TRUE, it needs to be
+ * built later on; otherwise it's built immediately.
  *
  * "tablespaceOid" is the tablespace to use for this index.
  */
 Oid
-index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
-							   Oid tablespaceOid, const char *newName)
+index_create_copy(Relation heapRelation, bool concurrently,
+				  Oid oldIndexId, Oid tablespaceOid, const char *newName)
 {
 	Relation	indexRelation;
 	IndexInfo  *oldInfo,
@@ -1318,6 +1318,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 	List	   *indexColNames = NIL;
 	List	   *indexExprs = NIL;
 	List	   *indexPreds = NIL;
+	int			flags = 0;
 
 	indexRelation = index_open(oldIndexId, RowExclusiveLock);
 
@@ -1328,7 +1329,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 	 * Concurrent build of an index with exclusion constraints is not
 	 * supported.
 	 */
-	if (oldInfo->ii_ExclusionOps != NULL)
+	if (oldInfo->ii_ExclusionOps != NULL && concurrently)
 		ereport(ERROR,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("concurrent index creation for exclusion constraints is not supported")));
@@ -1384,9 +1385,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 	}
 
 	/*
-	 * Build the index information for the new index.  Note that rebuild of
-	 * indexes with exclusion constraints is not supported, hence there is no
-	 * need to fill all the ii_Exclusion* fields.
+	 * Build the index information for the new index.
 	 */
 	newInfo = makeIndexInfo(oldInfo->ii_NumIndexAttrs,
 							oldInfo->ii_NumIndexKeyAttrs,
@@ -1395,10 +1394,13 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 							indexPreds,
 							oldInfo->ii_Unique,
 							oldInfo->ii_NullsNotDistinct,
-							false,	/* not ready for inserts */
-							true,
+							!concurrently,	/* isready */
+							concurrently,	/* concurrent */
 							indexRelation->rd_indam->amsummarizing,
-							oldInfo->ii_WithoutOverlaps);
+							oldInfo->ii_WithoutOverlaps,
+							oldInfo->ii_ExclusionOps,
+							oldInfo->ii_ExclusionProcs,
+							oldInfo->ii_ExclusionStrats);
 
 	/*
 	 * Extract the list of column names and the column numbers for the new
@@ -1436,6 +1438,9 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 		stattargets[i].isnull = isnull;
 	}
 
+	if (concurrently)
+		flags = INDEX_CREATE_SKIP_BUILD | INDEX_CREATE_CONCURRENT;
+
 	/*
 	 * Now create the new index.
 	 *
@@ -1459,7 +1464,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId,
 							  indcoloptions->values,
 							  stattargets,
 							  reloptionsDatum,
-							  INDEX_CREATE_SKIP_BUILD | INDEX_CREATE_CONCURRENT,
+							  flags,
 							  0,
 							  true, /* allow table to be a system catalog? */
 							  false,	/* is_internal? */
@@ -2453,7 +2458,8 @@ BuildIndexInfo(Relation index)
 					   indexStruct->indisready,
 					   false,
 					   index->rd_indam->amsummarizing,
-					   indexStruct->indisexclusion && indexStruct->indisunique);
+					   indexStruct->indisexclusion && indexStruct->indisunique,
+					   NULL, NULL, NULL);
 
 	/* fill in attribute numbers */
 	for (i = 0; i < numAtts; i++)
@@ -2513,7 +2519,8 @@ BuildDummyIndexInfo(Relation index)
 					   indexStruct->indisready,
 					   false,
 					   index->rd_indam->amsummarizing,
-					   indexStruct->indisexclusion && indexStruct->indisunique);
+					   indexStruct->indisexclusion && indexStruct->indisunique,
+					   NULL, NULL, NULL);
 
 	/* fill in attribute numbers */
 	for (i = 0; i < numAtts; i++)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index dd593ccbc1c..83edab38760 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -244,7 +244,8 @@ CheckIndexCompatible(Oid oldId,
 	 */
 	indexInfo = makeIndexInfo(numberOfAttributes, numberOfAttributes,
 							  accessMethodId, NIL, NIL, false, false,
-							  false, false, amsummarizing, isWithoutOverlaps);
+							  false, false, amsummarizing, isWithoutOverlaps,
+							  NULL, NULL, NULL);
 	typeIds = palloc_array(Oid, numberOfAttributes);
 	collationIds = palloc_array(Oid, numberOfAttributes);
 	opclassIds = palloc_array(Oid, numberOfAttributes);
@@ -931,7 +932,8 @@ DefineIndex(ParseState *pstate,
 							  !concurrent,
 							  concurrent,
 							  amissummarizing,
-							  stmt->iswithoutoverlaps);
+							  stmt->iswithoutoverlaps,
+							  NULL, NULL, NULL);
 
 	typeIds = palloc_array(Oid, numberOfAttributes);
 	collationIds = palloc_array(Oid, numberOfAttributes);
@@ -3989,10 +3991,11 @@ ReindexRelationConcurrently(const ReindexStmt *stmt, Oid relationOid, const Rein
 			tablespaceid = indexRel->rd_rel->reltablespace;
 
 		/* Create new index definition based on given index */
-		newIndexId = index_concurrently_create_copy(heapRel,
-													idx->indexId,
-													tablespaceid,
-													concurrentName);
+		newIndexId = index_create_copy(heapRel,
+									   true,
+									   idx->indexId,
+									   tablespaceid,
+									   concurrentName);
 
 		/*
 		 * Now open the relation of the new index, a session-level lock is
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 3cd35c5c457..8d23aa917e5 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -834,7 +834,8 @@ IndexInfo *
 makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions,
 			  List *predicates, bool unique, bool nulls_not_distinct,
 			  bool isready, bool concurrent, bool summarizing,
-			  bool withoutoverlaps)
+			  bool withoutoverlaps, Oid *exclusion_ops, Oid *exclusion_procs,
+			  uint16 *exclusion_strats)
 {
 	IndexInfo  *n = makeNode(IndexInfo);
 
@@ -863,9 +864,9 @@ makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions,
 	n->ii_PredicateState = NULL;
 
 	/* exclusion constraints */
-	n->ii_ExclusionOps = NULL;
-	n->ii_ExclusionProcs = NULL;
-	n->ii_ExclusionStrats = NULL;
+	n->ii_ExclusionOps = exclusion_ops;
+	n->ii_ExclusionProcs = exclusion_procs;
+	n->ii_ExclusionStrats = exclusion_strats;
 
 	/* speculative inserts */
 	n->ii_UniqueOps = NULL;
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index 36b70689254..56a064ef444 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -101,10 +101,9 @@ extern Oid	index_create(Relation heapRelation,
 #define	INDEX_CONSTR_CREATE_REMOVE_OLD_DEPS	(1 << 4)
 #define	INDEX_CONSTR_CREATE_WITHOUT_OVERLAPS (1 << 5)
 
-extern Oid	index_concurrently_create_copy(Relation heapRelation,
-										   Oid oldIndexId,
-										   Oid tablespaceOid,
-										   const char *newName);
+extern Oid	index_create_copy(Relation heapRelation, bool concurrently,
+							  Oid oldIndexId, Oid tablespaceOid,
+							  const char *newName);
 
 extern void index_concurrently_build(Oid heapRelationId,
 									 Oid indexRelationId);
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index bf54d39feb0..40ec249a7a1 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -99,7 +99,9 @@ extern IndexInfo *makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid,
 								List *expressions, List *predicates,
 								bool unique, bool nulls_not_distinct,
 								bool isready, bool concurrent,
-								bool summarizing, bool withoutoverlaps);
+								bool summarizing, bool withoutoverlaps,
+								Oid *exclusion_ops, Oid *exclusion_procs,
+								uint16 *exclusion_strats);
 
 extern Node *makeStringConst(char *str, int location);
 extern DefElem *makeDefElem(char *name, Node *arg, int location);
-- 
2.47.3


--fot5plqmb33ey33c
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v44b-0002-Do-not-dereference-varattrib_4b-in-VARSIZE_4B.patch"



^ permalink  raw  reply  [nested|flat] 57+ messages in thread

end of thread, other threads:[~2026-03-24 18:02 UTC | newest]

Thread overview: 57+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2026-03-24 18:02 [PATCH v44b 1/4] Make index_concurrently_create_copy more general Álvaro Herrera <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox