public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 7/7] Make Append node async-aware.
57+ messages / 2 participants
[nested] [flat]

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Thu_Jul_21_18_50_07_2016_597)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Thu_Jul_21_18_50_07_2016_597)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 67 +++++++++++++++++++++++++++++++++++++--
 src/include/nodes/execnodes.h     |  2 ++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..9a4063a 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,6 +199,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
+	int stopplan = node->as_whichplan;
+
 	for (;;)
 	{
 		PlanState  *subnode;
@@ -207,7 +214,36 @@ ExecAppend(AppendState *node)
 		/*
 		 * get a tuple from the subplan
 		 */
-		result = ExecProcNode(subnode);
+		Assert(!node->as_finished[node->as_whichplan]);
+
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				/* Move to the next living node */
+				do
+				{
+					node->as_whichplan = 
+						(node->as_whichplan + 1) %  node->as_nplans;
+				} while (node->as_whichplan != stopplan &&
+						 node->as_finished[node->as_whichplan]);
+
+				/* No node is ready yet, return as not-ready */
+				if (node->as_whichplan == stopplan)
+					return;
+
+				/* Try the next node */
+				continue;
+			}
+
+			/* If not async, immediately wait for this subnode */
+			ExecAsyncWaitForNode(subnode);
+		}				
+
+		result = ExecConsumeResult((PlanState *) subnode);
 
 		if (!TupIsNull(result))
 		{
@@ -220,6 +256,31 @@ ExecAppend(AppendState *node)
 			return;
 		}
 
+		if (node->as_async)
+		{
+			node->as_finished[node->as_whichplan] = true;
+			stopplan = node->as_whichplan;
+
+			/* Find the next living subnode */
+			do
+			{
+				node->as_whichplan =
+					(node->as_whichplan + 1) % node->as_nplans;
+			} while (node->as_whichplan != stopplan &&
+					 node->as_finished[node->as_whichplan]);
+
+			if (node->as_whichplan != stopplan)
+			{
+				stopplan = node->as_whichplan;
+				continue;
+			}
+
+			/* All subnodes are exhausted. Finish this node. */
+			ExecReturnTuple(&node->ps,
+							ExecClearTuple(node->ps.ps_ResultTupleSlot));
+			return;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
@@ -277,6 +338,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b72decc..b0a86c5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1177,6 +1177,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
2.9.2


----Next_Part(Mon_Aug_29_17_08_36_2016_213)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Mon_Aug_29_17_08_36_2016_213)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH 7/7] Make Append node async-aware.
@ 2016-06-28 09:52 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Kyotaro Horiguchi @ 2016-06-28 09:52 UTC (permalink / raw)

Change append node to be capable to handle asynchronous children
properly. As soon as it receives !async_ready from a child, it moves
to the next child and if no child is ready, it sleeps until at least
one of them become ready.
---
 src/backend/executor/nodeAppend.c | 94 ++++++++++++++++++++++++++++++++-------
 src/include/nodes/execnodes.h     |  2 +
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index e0ce8c6..1c0d26e 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -58,6 +58,7 @@
 #include "postgres.h"
 
 #include "executor/execdebug.h"
+#include "executor/execAsync.h"
 #include "executor/nodeAppend.h"
 
 static bool exec_append_initialize_next(AppendState *appendstate);
@@ -121,6 +122,7 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 {
 	AppendState *appendstate = makeNode(AppendState);
 	PlanState **appendplanstates;
+	bool	   *finished;
 	int			nplans;
 	int			i;
 	ListCell   *lc;
@@ -134,14 +136,17 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 	nplans = list_length(node->appendplans);
 
 	appendplanstates = (PlanState **) palloc0(nplans * sizeof(PlanState *));
-
+	finished = (bool *) palloc0(nplans * sizeof(bool));
+	
 	/*
 	 * create new AppendState for our append node
 	 */
 	appendstate->ps.plan = (Plan *) node;
 	appendstate->ps.state = estate;
 	appendstate->appendplans = appendplanstates;
+	appendstate->as_finished = finished;
 	appendstate->as_nplans = nplans;
+	appendstate->as_async = ((eflags & EXEC_FLAG_BACKWARD) == 0);
 
 	/*
 	 * Miscellaneous initialization
@@ -194,49 +199,104 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
 void
 ExecAppend(AppendState *node)
 {
-	for (;;)
+	int n_notready = 0;
+	PlanState  *subnode;
+	int i, n;
+
+	n = node->as_whichplan;
+
+	for (i = 0 ; i < node->as_nplans ; i++)
 	{
-		PlanState  *subnode;
 		TupleTableSlot *result;
 
+		if (node->as_async)
+		{
+			if (n >= node->as_nplans)
+				n = 0;
+
+			if (node->as_finished[n])
+			{
+				n++;
+				continue;
+			}
+		}
+
 		/*
 		 * figure out which subplan we are currently processing
 		 */
-		subnode = node->appendplans[node->as_whichplan];
+		subnode = node->appendplans[n];
 
 		/*
-		 * get a tuple from the subplan
+		 * execute the subplan to get a result if it is not ready yet
 		 */
-		result = ExecProcNode(subnode);
+		if (!subnode->result_ready)
+			ExecExecuteNode(subnode);
+
+		/*
+		 * This subnode is not ready yet when asynchrony is not allowed,
+		 * immediately wait for this subnode.
+		 */
+		if (!subnode->result_ready)
+		{
+			if (node->as_async)
+			{
+				n_notready++;
+				n++;
+				continue;
+			}
+			ExecAsyncWaitForNode(subnode);
+		}
+
+		Assert(subnode->result_ready);
+
+		result = ExecConsumeResult((PlanState *)subnode);
 
 		if (!TupIsNull(result))
 		{
 			/*
-			 * If the subplan gave us something then return it as-is. We do
-			 * NOT make use of the result slot that was set up in
-			 * ExecInitAppend; there's no need for it.
+			 * If the subplan gave us something then return it
+			 * as-is. We do NOT make use of the result slot that was
+			 * set up in ExecInitAppend; there's no need for it.
 			 */
+			node->as_whichplan = n;
 			ExecReturnTuple(&node->ps, result);
 			return;
 		}
 
+		/* Tuple has been exhausted on this subnode */
+		if (node->as_async)
+		{
+			node->as_finished[n++] = true;
+			continue;
+		}
+
 		/*
 		 * Go on to the "next" subplan in the appropriate direction. If no
 		 * more subplans, return the empty slot set up for us by
 		 * ExecInitAppend.
 		 */
 		if (ScanDirectionIsForward(node->ps.state->es_direction))
-			node->as_whichplan++;
+		{
+			if (n >= node->as_nplans - 1)
+				break;
+			n++;
+		}
 		else
-			node->as_whichplan--;
-		if (!exec_append_initialize_next(node))
 		{
-			ExecReturnTuple(&node->ps,
-							ExecClearTuple(node->ps.ps_ResultTupleSlot));
-			return;
+			if (n == 0)
+				break;
+			n--;
 		}
+	}
 
-		/* Else loop back and try to get a tuple from the new subplan */
+	/*
+	 * We are finished if reached here and no subnodes are not-ready
+	 */
+	if (n_notready == 0)
+	{
+		node->as_whichplan = n;
+		ExecReturnTuple(&node->ps,
+						ExecClearTuple(node->ps.ps_ResultTupleSlot));
 	}
 }
 
@@ -277,6 +337,8 @@ ExecReScanAppend(AppendState *node)
 	{
 		PlanState  *subnode = node->appendplans[i];
 
+		node->as_finished[i] = false;
+
 		/*
 		 * ExecReScan doesn't know about my subplans, so I have to do
 		 * changed-parameter signaling myself.
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9121537..e4f2bb6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1170,6 +1170,8 @@ typedef struct AppendState
 {
 	PlanState	ps;				/* its first field is NodeTag */
 	PlanState **appendplans;	/* array of PlanStates for my inputs */
+	bool		as_async;		/* true to allow async execution */
+	bool	   *as_finished;	/* array of the running state of subplans */
 	int			as_nplans;
 	int			as_whichplan;
 } AppendState;
-- 
1.8.3.1


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gentbl.sql"

CREATE EXTENSION postgres_fdw;
CREATE TABLE pl (a int, b int);
CREATE TABLE cl1 (LIKE pl) INHERITS (pl);
CREATE TABLE cl2 (LIKE pl) INHERITS (pl);
CREATE TABLE cl3 (LIKE pl) INHERITS (pl);
CREATE TABLE cl4 (LIKE pl) INHERITS (pl);
INSERT INTO cl1 (SELECT a, a FROM generate_series(0000000, 0999999) a);
INSERT INTO cl2 (SELECT a, a FROM generate_series(1000000, 1999999) a);
INSERT INTO cl3 (SELECT a, a FROM generate_series(2000000, 2999999) a);
INSERT INTO cl4 (SELECT a, a FROM generate_series(3000000, 3999999) a);

CREATE TABLE t0 (LIKE pl);
INSERT INTO t0 (SELECT a, a FROM generate_series(0000000, 9999999) a);

CREATE SERVER sv0 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv3 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE SERVER sv4 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '/tmp', dbname 'postgres');
CREATE USER MAPPING FOR public SERVER sv0;
CREATE USER MAPPING FOR public SERVER sv1;
CREATE USER MAPPING FOR public SERVER sv2;
CREATE USER MAPPING FOR public SERVER sv3;
CREATE USER MAPPING FOR public SERVER sv4;

CREATE FOREIGN TABLE ft10 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft20 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft30 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft40 (a int, b int) SERVER sv0 OPTIONS (table_name 'cl4');
CREATE FOREIGN TABLE ft11 (a int, b int) SERVER sv1 OPTIONS (table_name 'cl1');
CREATE FOREIGN TABLE ft22 (a int, b int) SERVER sv2 OPTIONS (table_name 'cl2');
CREATE FOREIGN TABLE ft33 (a int, b int) SERVER sv3 OPTIONS (table_name 'cl3');
CREATE FOREIGN TABLE ft44 (a int, b int) SERVER sv4 OPTIONS (table_name 'cl4');

CREATE TABLE pf0 (LIKE pl);
ALTER FOREIGN TABLE ft10 INHERIT pf0;
ALTER FOREIGN TABLE ft20 INHERIT pf0;
ALTER FOREIGN TABLE ft30 INHERIT pf0;
ALTER FOREIGN TABLE ft40 INHERIT pf0;

CREATE table pf1 (LIKE pl);
ALTER FOREIGN TABLE ft11 INHERIT pf1;
ALTER FOREIGN TABLE ft22 INHERIT pf1;
ALTER FOREIGN TABLE ft33 INHERIT pf1;
ALTER FOREIGN TABLE ft44 INHERIT pf1;

ANALYZE;

----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="testrun.sh"

#! /bin/bash

function do_test() {
	echo $1
	for i in $(seq 1 10); do psql postgres -c "set log_min_duration_statement = 0; set client_min_messages=log; select sum(a) from $1"; done | grep LOG
}

for i in "t0" "pl" "pf0" "pf1";
do
  do_test $i
done


----Next_Part(Wed_Jul_06_16_29_14_2016_599)--
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

----Next_Part(Wed_Jul_06_16_29_14_2016_599)----






^ permalink  raw  reply  [nested|flat] 57+ messages in thread

* [PATCH v43 7/7] Teach snapshot builder to skip transactions running REPACK (CONCURRENTLY).
@ 2026-03-17 19:22 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 57+ messages in thread

From: Antonin Houska @ 2026-03-17 19:22 UTC (permalink / raw)

During logical decoding, we need to know if particular transaction performed
catalog changes because catalog information is needed to construct heap
tuples. To be sure that we have enough information of each transaction, the
logical decoding cannot start before all the already running transactions have
completed.

The problem with REPACK (CONCURRENTLY) is that it has XID assigned and writes
WAL records marked with it. Thus if another backend runs REPACK (CONCURRENTLY)
and tries to setup the logical decoding, it has to wait for the completion of
all the other transactions involved in REPACK (CONCURRENTLY).

However, REPACK (CONCURRENTLY) does not perform any catalog changes relevant
to logical decoding, so the other backends executing this command can ignore
it. This patch implements it by adding information about transactions
executing the command to the xl_running_xacts WAL record, and by teaching the
snapshot builder to use the information.
---
 src/backend/access/rmgrdesc/standbydesc.c   | 15 ++++-
 src/backend/access/transam/xlog.c           |  2 +
 src/backend/commands/cluster.c              | 16 +++++
 src/backend/replication/logical/snapbuild.c | 38 ++++++++---
 src/backend/storage/ipc/procarray.c         | 73 +++++++++++++++++++--
 src/backend/storage/ipc/standby.c           |  6 +-
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/proc.h                  |  3 +-
 src/include/storage/standby.h               |  3 +
 src/include/storage/standbydefs.h           |  2 +
 10 files changed, 138 insertions(+), 22 deletions(-)

diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..79c052d3607 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -21,10 +21,11 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 {
 	int			i;
 
-	appendStringInfo(buf, "nextXid %u latestCompletedXid %u oldestRunningXid %u",
+	appendStringInfo(buf, "nextXid %u latestCompletedXid %u oldestRunningXid %u oldestRunningXid %u",
 					 xlrec->nextXid,
 					 xlrec->latestCompletedXid,
-					 xlrec->oldestRunningXid);
+					 xlrec->oldestRunningXid,
+					 xlrec->oldestRunningXidLogical);
 	if (xlrec->xcnt > 0)
 	{
 		appendStringInfo(buf, "; %d xacts:", xlrec->xcnt);
@@ -41,6 +42,16 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	if (xlrec->xcnt_repack > 0)
+	{
+		TransactionId	*xids_repack;
+
+		appendStringInfo(buf, "; %d xacts_repack:", xlrec->xcnt_repack);
+		xids_repack = xlrec->xids + xlrec->xcnt + xlrec->subxcnt;
+		for (i = 0; i < xlrec->xcnt_repack; i++)
+			appendStringInfo(buf, " %u", xids_repack[i]);
+	}
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5c9a34374d..d050b0b1444 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -5912,6 +5912,7 @@ StartupXLOG(void)
 				 * subxids are listed with their parent prepared transactions.
 				 */
 				running.xcnt = nxids;
+				running.xcnt_repack = 0;
 				running.subxcnt = 0;
 				running.subxid_status = SUBXIDS_IN_SUBTRANS;
 				running.nextXid = XidFromFullTransactionId(checkPoint.nextXid);
@@ -8477,6 +8478,7 @@ xlog_redo(XLogReaderState *record)
 			 * with their parent prepared transactions.
 			 */
 			running.xcnt = nxids;
+			running.xcnt_repack = 0;
 			running.subxcnt = 0;
 			running.subxid_status = SUBXIDS_IN_SUBTRANS;
 			running.nextXid = XidFromFullTransactionId(checkPoint.nextXid);
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 065316ee866..77c206ff944 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -1084,6 +1084,22 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose,
 
 	if (concurrent)
 	{
+		/*
+		 * Do not let other backends wait for our completion during their
+		 * setup of logical replication. Unlike logical replication publisher,
+		 * we will have XID assigned, so the other backends - whether
+		 * walsenders involved in logical replication or regular backends
+		 * executing also REPACK (CONCURRENTLY) - would have to wait for our
+		 * completion before they can build their initial snapshot. It is
+		 * o.k. for any decoding backend to ignore us because we do not change
+		 * tuple descriptor of any table, and the data changes we write should
+		 * not be decoded by other backends.
+		 */
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+		MyProc->statusFlags |= PROC_IN_REPACK;
+		ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
+		LWLockRelease(ProcArrayLock);
+
 		/*
 		 * The worker needs to be member of the locking group we're the leader
 		 * of. We ought to become the leader before the worker starts. The
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 9cf499ce7c6..fbdd4600a2b 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -1172,7 +1172,7 @@ SnapBuildProcessRunningXacts(SnapBuild *builder, XLogRecPtr lsn, xl_running_xact
 	 * xmin, which looks odd but is correct and actually more efficient, since
 	 * we hit fast paths in heapam_visibility.c.
 	 */
-	builder->xmin = running->oldestRunningXid;
+	builder->xmin = running->oldestRunningXidLogical;
 
 	/* Remove transactions we don't need to keep track off anymore */
 	SnapBuildPurgeOlderTxn(builder);
@@ -1188,9 +1188,9 @@ SnapBuildProcessRunningXacts(SnapBuild *builder, XLogRecPtr lsn, xl_running_xact
 	 */
 	xmin = ReorderBufferGetOldestXmin(builder->reorder);
 	if (xmin == InvalidTransactionId)
-		xmin = running->oldestRunningXid;
+		xmin = running->oldestRunningXidLogical;
 	elog(DEBUG3, "xmin: %u, xmax: %u, oldest running: %u, oldest xmin: %u",
-		 builder->xmin, builder->xmax, running->oldestRunningXid, xmin);
+		 builder->xmin, builder->xmax, running->oldestRunningXidLogical, xmin);
 	LogicalIncreaseXminForSlot(lsn, xmin);
 
 	/*
@@ -1275,14 +1275,14 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn
 	 * have all necessary catalog rows anymore.
 	 */
 	if (TransactionIdIsNormal(builder->initial_xmin_horizon) &&
-		NormalTransactionIdPrecedes(running->oldestRunningXid,
+		NormalTransactionIdPrecedes(running->oldestRunningXidLogical,
 									builder->initial_xmin_horizon))
 	{
 		ereport(DEBUG1,
 				errmsg_internal("skipping snapshot at %X/%08X while building logical decoding snapshot, xmin horizon too low",
 								LSN_FORMAT_ARGS(lsn)),
 				errdetail_internal("initial xmin horizon of %u vs the snapshot's %u",
-								   builder->initial_xmin_horizon, running->oldestRunningXid));
+								   builder->initial_xmin_horizon, running->oldestRunningXidLogical));
 
 
 		SnapBuildWaitSnapshot(running, builder->initial_xmin_horizon);
@@ -1299,7 +1299,7 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn
 	 * NB: We might have already started to incrementally assemble a snapshot,
 	 * so we need to be careful to deal with that.
 	 */
-	if (running->oldestRunningXid == running->nextXid)
+	if (running->oldestRunningXidLogical == running->nextXid)
 	{
 		if (!XLogRecPtrIsValid(builder->start_decoding_at) ||
 			builder->start_decoding_at <= lsn)
@@ -1378,14 +1378,14 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn
 	/*
 	 * c) transition from BUILDING_SNAPSHOT to FULL_SNAPSHOT.
 	 *
-	 * In BUILDING_SNAPSHOT state, and this xl_running_xacts' oldestRunningXid
+	 * In BUILDING_SNAPSHOT state, and this xl_running_xacts' oldestRunningXidLogical
 	 * is >= than nextXid from when we switched to BUILDING_SNAPSHOT.  This
 	 * means all transactions starting afterwards have enough information to
 	 * be decoded.  Switch to FULL_SNAPSHOT.
 	 */
 	else if (builder->state == SNAPBUILD_BUILDING_SNAPSHOT &&
 			 TransactionIdPrecedesOrEquals(builder->next_phase_at,
-										   running->oldestRunningXid))
+										   running->oldestRunningXidLogical))
 	{
 		builder->state = SNAPBUILD_FULL_SNAPSHOT;
 		builder->next_phase_at = running->nextXid;
@@ -1402,14 +1402,14 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn
 	/*
 	 * c) transition from FULL_SNAPSHOT to CONSISTENT.
 	 *
-	 * In FULL_SNAPSHOT state, and this xl_running_xacts' oldestRunningXid is
+	 * In FULL_SNAPSHOT state, and this xl_running_xacts' oldestRunningXidLogical is
 	 * >= than nextXid from when we switched to FULL_SNAPSHOT.  This means all
 	 * transactions that are currently in progress have a catalog snapshot,
 	 * and all their changes have been collected.  Switch to CONSISTENT.
 	 */
 	else if (builder->state == SNAPBUILD_FULL_SNAPSHOT &&
 			 TransactionIdPrecedesOrEquals(builder->next_phase_at,
-										   running->oldestRunningXid))
+										   running->oldestRunningXidLogical))
 	{
 		builder->state = SNAPBUILD_CONSISTENT;
 		builder->next_phase_at = InvalidTransactionId;
@@ -1459,6 +1459,24 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 		if (TransactionIdFollows(xid, cutoff))
 			continue;
 
+		/* Do not wait for transactions running REPACK (CONCURRENTLY). */
+		if (running->xcnt_repack > 0)
+		{
+			TransactionId	*xids_repack;
+			int		i;
+
+			xids_repack = running->xids + running->xcnt + running->subxcnt;
+
+			for (i = 0; i < running->xcnt_repack; i++)
+			{
+				if (xid == xids_repack[i])
+					break;
+			}
+			/* Found? */
+			if (i < running->xcnt_repack)
+				continue;
+		}
+
 		XactLockTableWait(xid, NULL, NULL, XLTW_None);
 	}
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 0f913897acc..f1a03e6a754 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2643,15 +2643,24 @@ GetRunningTransactionData(void)
 	RunningTransactions CurrentRunningXacts = &CurrentRunningXactsData;
 	TransactionId latestCompletedXid;
 	TransactionId oldestRunningXid;
+	TransactionId oldestRunningXidLogical;
 	TransactionId oldestDatabaseRunningXid;
 	TransactionId *xids;
 	int			index;
-	int			count;
+	int			count, count_repack;
 	int			subcount;
 	bool		suboverflowed;
+	TransactionId	*xids_repack = NULL;
+	bool		logical_decoding_enabled = IsLogicalDecodingEnabled();
 
 	Assert(!RecoveryInProgress());
 
+	/*
+	 * TODO Consider a GUC to reserve certain amount of replication slots for
+	 * REPACK (CONCURRENTLY) and using it here.
+	 */
+#define		MAX_REPACK_XIDS		16
+
 	/*
 	 * Allocating space for maxProcs xids is usually overkill; numProcs would
 	 * be sufficient.  But it seems better to do the malloc while not holding
@@ -2663,11 +2672,13 @@ GetRunningTransactionData(void)
 	 */
 	if (CurrentRunningXacts->xids == NULL)
 	{
+		int		nrepack = logical_decoding_enabled ? MAX_REPACK_XIDS : 0;
+
 		/*
 		 * First call
 		 */
 		CurrentRunningXacts->xids = (TransactionId *)
-			malloc(TOTAL_MAX_CACHED_SUBXIDS * sizeof(TransactionId));
+			malloc((TOTAL_MAX_CACHED_SUBXIDS + nrepack) * sizeof(TransactionId));
 		if (CurrentRunningXacts->xids == NULL)
 			ereport(ERROR,
 					(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -2676,7 +2687,10 @@ GetRunningTransactionData(void)
 
 	xids = CurrentRunningXacts->xids;
 
-	count = subcount = 0;
+	if (logical_decoding_enabled)
+		xids_repack = palloc_array(TransactionId, MAX_REPACK_XIDS);
+
+	count = subcount = count_repack = 0;
 	suboverflowed = false;
 
 	/*
@@ -2688,7 +2702,7 @@ GetRunningTransactionData(void)
 
 	latestCompletedXid =
 		XidFromFullTransactionId(TransamVariables->latestCompletedXid);
-	oldestDatabaseRunningXid = oldestRunningXid =
+	oldestDatabaseRunningXid = oldestRunningXid = oldestRunningXidLogical =
 		XidFromFullTransactionId(TransamVariables->nextXid);
 
 	/*
@@ -2697,6 +2711,8 @@ GetRunningTransactionData(void)
 	for (index = 0; index < arrayP->numProcs; index++)
 	{
 		TransactionId xid;
+		int			pgprocno;
+		PGPROC	   *proc;
 
 		/* Fetch xid just once - see GetNewTransactionId */
 		xid = UINT32_ACCESS_ONCE(other_xids[index]);
@@ -2716,6 +2732,21 @@ GetRunningTransactionData(void)
 		if (TransactionIdPrecedes(xid, oldestRunningXid))
 			oldestRunningXid = xid;
 
+		if (logical_decoding_enabled &&
+			TransactionIdPrecedes(xid, oldestRunningXidLogical))
+		{
+			/*
+			 * Backends running REPACK concurrently need to be excluded from
+			 * oldestRunningXidLogical, otherwise the snapshot builder cannot
+			 * proceed in building the initial snapshot.
+			 */
+			pgprocno = arrayP->pgprocnos[index];
+			proc = &allProcs[pgprocno];
+
+			if ((proc->statusFlags & PROC_IN_REPACK) == 0)
+				oldestRunningXidLogical = xid;
+		}
+
 		/*
 		 * Also, update the oldest running xid within the current database. As
 		 * fetching pgprocno and PGPROC could cause cache misses, we do cheap
@@ -2723,8 +2754,8 @@ GetRunningTransactionData(void)
 		 */
 		if (TransactionIdPrecedes(xid, oldestDatabaseRunningXid))
 		{
-			int			pgprocno = arrayP->pgprocnos[index];
-			PGPROC	   *proc = &allProcs[pgprocno];
+			pgprocno = arrayP->pgprocnos[index];
+			proc = &allProcs[pgprocno];
 
 			if (proc->databaseId == MyDatabaseId)
 				oldestDatabaseRunningXid = xid;
@@ -2742,6 +2773,19 @@ GetRunningTransactionData(void)
 		 */
 
 		xids[count++] = xid;
+
+		/*
+		 * Collect XIDSs of transactions running REPACK (CONCURRENTLY).
+		 */
+		if (logical_decoding_enabled &&
+			count_repack < MAX_REPACK_XIDS)
+		{
+			pgprocno = arrayP->pgprocnos[index];
+			proc = &allProcs[pgprocno];
+
+			if (proc->statusFlags & PROC_IN_REPACK)
+				xids_repack[count_repack++] = xid;
+		}
 	}
 
 	/*
@@ -2782,6 +2826,19 @@ GetRunningTransactionData(void)
 		}
 	}
 
+	/*
+	 * Append the XIDs running REPACK (CONCURRENTLY), if any.
+	 *
+	 * XXX Should we sort the array and use bsearch() when using it?
+	 */
+	if (count_repack > 0)
+	{
+		for (int i = 0; i < count_repack; i++)
+			xids[count++] = xids_repack[i];
+	}
+	if (xids_repack)
+		pfree(xids_repack);
+
 	/*
 	 * It's important *not* to include the limits set by slots here because
 	 * snapbuild.c uses oldestRunningXid to manage its xmin horizon. If those
@@ -2791,11 +2848,13 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
-	CurrentRunningXacts->xcnt = count - subcount;
+	CurrentRunningXacts->xcnt = count - subcount - count_repack;
 	CurrentRunningXacts->subxcnt = subcount;
+	CurrentRunningXacts->xcnt_repack = count_repack;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
 	CurrentRunningXacts->nextXid = XidFromFullTransactionId(TransamVariables->nextXid);
 	CurrentRunningXacts->oldestRunningXid = oldestRunningXid;
+	CurrentRunningXacts->oldestRunningXidLogical = oldestRunningXidLogical;
 	CurrentRunningXacts->oldestDatabaseRunningXid = oldestDatabaseRunningXid;
 	CurrentRunningXacts->latestCompletedXid = latestCompletedXid;
 
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index f3ad90c7c7a..5c4121fcbc8 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,7 @@ standby_redo(XLogReaderState *record)
 		RunningTransactionsData running;
 
 		running.xcnt = xlrec->xcnt;
+		running.xcnt_repack = xlrec->xcnt_repack;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
 		running.nextXid = xlrec->nextXid;
@@ -1358,10 +1359,12 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	XLogRecPtr	recptr;
 
 	xlrec.xcnt = CurrRunningXacts->xcnt;
+	xlrec.xcnt_repack = CurrRunningXacts->xcnt_repack;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
 	xlrec.nextXid = CurrRunningXacts->nextXid;
 	xlrec.oldestRunningXid = CurrRunningXacts->oldestRunningXid;
+	xlrec.oldestRunningXidLogical = CurrRunningXacts->oldestRunningXidLogical;
 	xlrec.latestCompletedXid = CurrRunningXacts->latestCompletedXid;
 
 	/* Header */
@@ -1372,7 +1375,8 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	/* array of TransactionIds */
 	if (xlrec.xcnt > 0)
 		XLogRegisterData(CurrRunningXacts->xids,
-						 (xlrec.xcnt + xlrec.subxcnt) * sizeof(TransactionId));
+						 (xlrec.xcnt + xlrec.xcnt_repack + xlrec.subxcnt) *
+						 sizeof(TransactionId));
 
 	recptr = XLogInsert(RM_STANDBY_ID, XLOG_RUNNING_XACTS);
 
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 629ac3a7d3e..1730b07810f 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11D	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index bf3094f0f7d..120370f459d 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -67,10 +67,11 @@ struct XidCache
 #define		PROC_AFFECTS_ALL_HORIZONS	0x20	/* this proc's xmin must be
 												 * included in vacuum horizons
 												 * in all databases */
+#define		PROC_IN_REPACK				0x40	/* currently REPACK (CONCURRENTLY) */
 
 /* flags reset at EOXact */
 #define		PROC_VACUUM_STATE_MASK \
-	(PROC_IN_VACUUM | PROC_IN_SAFE_IC | PROC_VACUUM_FOR_WRAPAROUND)
+	(PROC_IN_VACUUM | PROC_IN_SAFE_IC | PROC_VACUUM_FOR_WRAPAROUND | PROC_IN_REPACK)
 
 /*
  * Xmin-related flags. Make sure any flags that affect how the process' Xmin
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index c63a4f2cc6a..0bd28e270f1 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -124,10 +124,13 @@ typedef enum
 typedef struct RunningTransactionsData
 {
 	int			xcnt;			/* # of xact ids in xids[] */
+	int			xcnt_repack;	/* # of xacts running REPACK
+								 * (CONCURRENTLY). */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
 	TransactionId nextXid;		/* xid from TransamVariables->nextXid */
 	TransactionId oldestRunningXid; /* *not* oldestXmin */
+	TransactionId oldestRunningXidLogical;
 	TransactionId oldestDatabaseRunningXid; /* same as above, but within the
 											 * current database */
 	TransactionId latestCompletedXid;	/* so we can set xmax */
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..edad609fa9a 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -47,10 +47,12 @@ typedef struct xl_standby_locks
 typedef struct xl_running_xacts
 {
 	int			xcnt;			/* # of xact ids in xids[] */
+	int			xcnt_repack;	/* # of xacts running REPACK (CONCURRENTLY) */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
 	TransactionId nextXid;		/* xid from TransamVariables->nextXid */
 	TransactionId oldestRunningXid; /* *not* oldestXmin */
+	TransactionId oldestRunningXidLogical;
 	TransactionId latestCompletedXid;	/* so we can set xmax */
 
 	TransactionId xids[FLEXIBLE_ARRAY_MEMBER];
-- 
2.47.3


--nzintiedl6o4kcyp--





^ permalink  raw  reply  [nested|flat] 57+ messages in thread

end of thread, other threads:[~2026-03-17 19:22 UTC | newest]

Thread overview: 57+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2016-06-28 09:52 [PATCH 7/7] Make Append node async-aware. Kyotaro Horiguchi <[email protected]>
2026-03-17 19:22 [PATCH v43 7/7] Teach snapshot builder to skip transactions running REPACK (CONCURRENTLY). Antonin Houska <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox