public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 2/3] Add operator <->(box, point) to GiST box_ops
108+ messages / 5 participants
[nested] [flat]

* [PATCH 2/3] Add operator <->(box, point) to GiST box_ops
@ 2019-03-07 20:49 Nikita Glukhov <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Nikita Glukhov @ 2019-03-07 20:49 UTC (permalink / raw)

---
 doc/src/sgml/gist.sgml             |  1 +
 src/backend/access/gist/gistproc.c | 53 ++++++++++++++++++--------
 src/include/catalog/pg_amop.dat    |  3 ++
 src/include/catalog/pg_amproc.dat  |  2 +
 src/include/catalog/pg_proc.dat    |  4 ++
 src/test/regress/expected/gist.out | 76 ++++++++++++++++++++++++++++++++++++++
 src/test/regress/sql/gist.sql      | 16 ++++++++
 7 files changed, 140 insertions(+), 15 deletions(-)

diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index 44a3b2c..e903319 100644
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -83,6 +83,7 @@
        <literal>~=</literal>
       </entry>
       <entry>
+       <literal>&lt;-&gt;</literal>
       </entry>
      </row>
      <row>
diff --git a/src/backend/access/gist/gistproc.c b/src/backend/access/gist/gistproc.c
index c9da230..89643b4 100644
--- a/src/backend/access/gist/gistproc.c
+++ b/src/backend/access/gist/gistproc.c
@@ -1464,26 +1464,13 @@ gist_point_distance(PG_FUNCTION_ARGS)
 	PG_RETURN_FLOAT8(distance);
 }
 
-/*
- * The inexact GiST distance method for geometric types that store bounding
- * boxes.
- *
- * Compute lossy distance from point to index entries.  The result is inexact
- * because index entries are bounding boxes, not the exact shapes of the
- * indexed geometric types.  We use distance from point to MBR of index entry.
- * This is a lower bound estimate of distance from point to indexed geometric
- * type.
- */
 static float8
-gist_bbox_distance(GISTENTRY *entry, Datum query,
-				   StrategyNumber strategy, bool *recheck)
+gist_box_distance_helper(GISTENTRY *entry, Datum query,
+						 StrategyNumber strategy)
 {
 	float8		distance;
 	StrategyNumber strategyGroup = strategy / GeoStrategyNumberOffset;
 
-	/* Bounding box distance is always inexact. */
-	*recheck = true;
-
 	switch (strategyGroup)
 	{
 		case PointStrategyNumberGroup:
@@ -1500,6 +1487,42 @@ gist_bbox_distance(GISTENTRY *entry, Datum query,
 }
 
 Datum
+gist_box_distance(PG_FUNCTION_ARGS)
+{
+	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
+	Datum		query = PG_GETARG_DATUM(1);
+	StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2);
+
+	/* Oid subtype = PG_GETARG_OID(3); */
+	/* bool	   *recheck = (bool *) PG_GETARG_POINTER(4); */
+	float8		distance;
+
+	distance = gist_box_distance_helper(entry, query, strategy);
+
+	PG_RETURN_FLOAT8(distance);
+}
+
+/*
+ * The inexact GiST distance method for geometric types that store bounding
+ * boxes.
+ *
+ * Compute lossy distance from point to index entries.  The result is inexact
+ * because index entries are bounding boxes, not the exact shapes of the
+ * indexed geometric types.  We use distance from point to MBR of index entry.
+ * This is a lower bound estimate of distance from point to indexed geometric
+ * type.
+ */
+static float8
+gist_bbox_distance(GISTENTRY *entry, Datum query,
+				   StrategyNumber strategy, bool *recheck)
+{
+	/* Bounding box distance is always inexact. */
+	*recheck = true;
+
+	return gist_box_distance_helper(entry, query, strategy);
+}
+
+Datum
 gist_circle_distance(PG_FUNCTION_ARGS)
 {
 	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat
index 0ab95d8..8e74f86 100644
--- a/src/include/catalog/pg_amop.dat
+++ b/src/include/catalog/pg_amop.dat
@@ -1081,6 +1081,9 @@
   amopstrategy => '13', amopopr => '~(box,box)', amopmethod => 'gist' },
 { amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box',
   amopstrategy => '14', amopopr => '@(box,box)', amopmethod => 'gist' },
+{ amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'point',
+  amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(box,point)',
+  amopmethod => 'gist', amopsortfamily => 'btree/float_ops' },
 
 # gist point_ops
 { amopfamily => 'gist/point_ops', amoplefttype => 'point',
diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat
index 020b741..5567b7e 100644
--- a/src/include/catalog/pg_amproc.dat
+++ b/src/include/catalog/pg_amproc.dat
@@ -419,6 +419,8 @@
   amprocrighttype => 'box', amprocnum => '6', amproc => 'gist_box_picksplit' },
 { amprocfamily => 'gist/box_ops', amproclefttype => 'box',
   amprocrighttype => 'box', amprocnum => '7', amproc => 'gist_box_same' },
+{ amprocfamily => 'gist/box_ops', amproclefttype => 'box',
+  amprocrighttype => 'box', amprocnum => '8', amproc => 'gist_box_distance' },
 { amprocfamily => 'gist/poly_ops', amproclefttype => 'polygon',
   amprocrighttype => 'polygon', amprocnum => '1',
   amproc => 'gist_poly_consistent' },
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 3800150..d6477a3 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7760,6 +7760,10 @@
 { oid => '2584', descr => 'GiST support',
   proname => 'gist_box_same', prorettype => 'internal',
   proargtypes => 'box box internal', prosrc => 'gist_box_same' },
+{ oid => '3998', descr => 'GiST support',
+  proname => 'gist_box_distance', prorettype => 'float8',
+  proargtypes => 'internal box int2 oid internal',
+  prosrc => 'gist_box_distance' },
 { oid => '2585', descr => 'GiST support',
   proname => 'gist_poly_consistent', prorettype => 'bool',
   proargtypes => 'internal polygon int2 oid internal',
diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out
index f5a2993..e8c2685 100644
--- a/src/test/regress/expected/gist.out
+++ b/src/test/regress/expected/gist.out
@@ -205,6 +205,82 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
  (6,6),(6,6)
 (21 rows)
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
 drop index gist_tbl_box_index;
 -- Test that an index-only scan is not chosen, when the query involves the
 -- circle column (the circle opclass does not support index-only scans).
diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql
index bae722f..406fb83 100644
--- a/src/test/regress/sql/gist.sql
+++ b/src/test/regress/sql/gist.sql
@@ -111,6 +111,22 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 -- execute the same
 select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
 drop index gist_tbl_box_index;
 
 -- Test that an index-only scan is not chosen, when the query involves the
-- 
2.7.4


--------------B01ADF66983FA504E371CA13
Content-Type: text/x-patch;
 name="0003-Add-box-point-distance-operator-to-SP-GiST-box_ops-v01.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="0003-Add-box-point-distance-operator-to-SP-GiST-box_ops-v01.";
 filename*1="patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH 2/3] Add operator <->(box, point) to GiST box_ops
@ 2019-03-07 20:49 Nikita Glukhov <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Nikita Glukhov @ 2019-03-07 20:49 UTC (permalink / raw)

---
 doc/src/sgml/gist.sgml             |  1 +
 src/backend/access/gist/gistproc.c | 49 +++++++++++++++---------
 src/include/catalog/pg_amop.dat    |  3 ++
 src/include/catalog/pg_amproc.dat  |  2 +
 src/include/catalog/pg_proc.dat    |  4 ++
 src/test/regress/expected/gist.out | 76 ++++++++++++++++++++++++++++++++++++++
 src/test/regress/sql/gist.sql      | 16 ++++++++
 7 files changed, 134 insertions(+), 17 deletions(-)

diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index 763b8cf..2b43dfc 100644
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -83,6 +83,7 @@
        <literal>~=</literal>
       </entry>
       <entry>
+       <literal>&lt;-&gt;</literal>
       </entry>
      </row>
      <row>
diff --git a/src/backend/access/gist/gistproc.c b/src/backend/access/gist/gistproc.c
index 1826b51..a668283 100644
--- a/src/backend/access/gist/gistproc.c
+++ b/src/backend/access/gist/gistproc.c
@@ -1464,26 +1464,13 @@ gist_point_distance(PG_FUNCTION_ARGS)
 	PG_RETURN_FLOAT8(distance);
 }
 
-/*
- * The inexact GiST distance method for geometric types that store bounding
- * boxes.
- *
- * Compute lossy distance from point to index entries.  The result is inexact
- * because index entries are bounding boxes, not the exact shapes of the
- * indexed geometric types.  We use distance from point to MBR of index entry.
- * This is a lower bound estimate of distance from point to indexed geometric
- * type.
- */
 static float8
-gist_bbox_distance(GISTENTRY *entry, Datum query,
-				   StrategyNumber strategy, bool *recheck)
+gist_box_distance_helper(GISTENTRY *entry, Datum query,
+						 StrategyNumber strategy)
 {
 	float8		distance;
 	StrategyNumber strategyGroup = strategy / GeoStrategyNumberOffset;
 
-	/* Bounding box distance is always inexact. */
-	*recheck = true;
-
 	switch (strategyGroup)
 	{
 		case PointStrategyNumberGroup:
@@ -1500,6 +1487,32 @@ gist_bbox_distance(GISTENTRY *entry, Datum query,
 }
 
 Datum
+gist_box_distance(PG_FUNCTION_ARGS)
+{
+	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
+	Datum		query = PG_GETARG_DATUM(1);
+	StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2);
+
+	/* Oid subtype = PG_GETARG_OID(3); */
+	/* bool	   *recheck = (bool *) PG_GETARG_POINTER(4); */
+	float8		distance;
+
+	distance = gist_box_distance_helper(entry, query, strategy);
+
+	PG_RETURN_FLOAT8(distance);
+}
+
+/*
+ * The inexact GiST distance methods for geometric types that store bounding
+ * boxes.
+ *
+ * Compute lossy distance from point to index entries.  The result is inexact
+ * because index entries are bounding boxes, not the exact shapes of the
+ * indexed geometric types.  We use distance from point to MBR of index entry.
+ * This is a lower bound estimate of distance from point to indexed geometric
+ * type.
+ */
+Datum
 gist_circle_distance(PG_FUNCTION_ARGS)
 {
 	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
@@ -1510,7 +1523,8 @@ gist_circle_distance(PG_FUNCTION_ARGS)
 	bool	   *recheck = (bool *) PG_GETARG_POINTER(4);
 	float8		distance;
 
-	distance = gist_bbox_distance(entry, query, strategy, recheck);
+	distance = gist_box_distance_helper(entry, query, strategy);
+	*recheck = true;
 
 	PG_RETURN_FLOAT8(distance);
 }
@@ -1526,7 +1540,8 @@ gist_poly_distance(PG_FUNCTION_ARGS)
 	bool	   *recheck = (bool *) PG_GETARG_POINTER(4);
 	float8		distance;
 
-	distance = gist_bbox_distance(entry, query, strategy, recheck);
+	distance = gist_box_distance_helper(entry, query, strategy);
+	*recheck = true;
 
 	PG_RETURN_FLOAT8(distance);
 }
diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat
index cf63eb7..ebc38ae 100644
--- a/src/include/catalog/pg_amop.dat
+++ b/src/include/catalog/pg_amop.dat
@@ -1081,6 +1081,9 @@
   amopstrategy => '13', amopopr => '~(box,box)', amopmethod => 'gist' },
 { amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box',
   amopstrategy => '14', amopopr => '@(box,box)', amopmethod => 'gist' },
+{ amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'point',
+  amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(box,point)',
+  amopmethod => 'gist', amopsortfamily => 'btree/float_ops' },
 
 # gist point_ops
 { amopfamily => 'gist/point_ops', amoplefttype => 'point',
diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat
index 020b741..5567b7e 100644
--- a/src/include/catalog/pg_amproc.dat
+++ b/src/include/catalog/pg_amproc.dat
@@ -419,6 +419,8 @@
   amprocrighttype => 'box', amprocnum => '6', amproc => 'gist_box_picksplit' },
 { amprocfamily => 'gist/box_ops', amproclefttype => 'box',
   amprocrighttype => 'box', amprocnum => '7', amproc => 'gist_box_same' },
+{ amprocfamily => 'gist/box_ops', amproclefttype => 'box',
+  amprocrighttype => 'box', amprocnum => '8', amproc => 'gist_box_distance' },
 { amprocfamily => 'gist/poly_ops', amproclefttype => 'polygon',
   amprocrighttype => 'polygon', amprocnum => '1',
   amproc => 'gist_poly_consistent' },
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index e0852b8..762c099 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7844,6 +7844,10 @@
 { oid => '2584', descr => 'GiST support',
   proname => 'gist_box_same', prorettype => 'internal',
   proargtypes => 'box box internal', prosrc => 'gist_box_same' },
+{ oid => '3998', descr => 'GiST support',
+  proname => 'gist_box_distance', prorettype => 'float8',
+  proargtypes => 'internal box int2 oid internal',
+  prosrc => 'gist_box_distance' },
 { oid => '2585', descr => 'GiST support',
   proname => 'gist_poly_consistent', prorettype => 'bool',
   proargtypes => 'internal polygon int2 oid internal',
diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out
index 0a43449..2234876 100644
--- a/src/test/regress/expected/gist.out
+++ b/src/test/regress/expected/gist.out
@@ -203,6 +203,82 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
  (6,6),(6,6)
 (21 rows)
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
 drop index gist_tbl_box_index;
 -- Test that an index-only scan is not chosen, when the query involves the
 -- circle column (the circle opclass does not support index-only scans).
diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql
index 657b195..b9d398e 100644
--- a/src/test/regress/sql/gist.sql
+++ b/src/test/regress/sql/gist.sql
@@ -109,6 +109,22 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 -- execute the same
 select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
 drop index gist_tbl_box_index;
 
 -- Test that an index-only scan is not chosen, when the query involves the
-- 
2.7.4


--------------C466D744B1EAB9FA85D22553
Content-Type: text/x-patch;
 name="0003-Add-operator-box-point-to-SP-GiST-box_ops-v03.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename="0003-Add-operator-box-point-to-SP-GiST-box_ops-v03.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH 2/3] Add operator <->(box, point) to GiST box_ops
@ 2019-03-07 20:49 Nikita Glukhov <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Nikita Glukhov @ 2019-03-07 20:49 UTC (permalink / raw)

---
 doc/src/sgml/gist.sgml             |  1 +
 src/backend/access/gist/gistproc.c | 53 ++++++++++++++++++--------
 src/include/catalog/pg_amop.dat    |  3 ++
 src/include/catalog/pg_amproc.dat  |  2 +
 src/include/catalog/pg_proc.dat    |  4 ++
 src/test/regress/expected/gist.out | 76 ++++++++++++++++++++++++++++++++++++++
 src/test/regress/sql/gist.sql      | 16 ++++++++
 7 files changed, 140 insertions(+), 15 deletions(-)

diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index 44a3b2c..e903319 100644
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -83,6 +83,7 @@
        <literal>~=</literal>
       </entry>
       <entry>
+       <literal>&lt;-&gt;</literal>
       </entry>
      </row>
      <row>
diff --git a/src/backend/access/gist/gistproc.c b/src/backend/access/gist/gistproc.c
index 1826b51..89bd656 100644
--- a/src/backend/access/gist/gistproc.c
+++ b/src/backend/access/gist/gistproc.c
@@ -1464,26 +1464,13 @@ gist_point_distance(PG_FUNCTION_ARGS)
 	PG_RETURN_FLOAT8(distance);
 }
 
-/*
- * The inexact GiST distance method for geometric types that store bounding
- * boxes.
- *
- * Compute lossy distance from point to index entries.  The result is inexact
- * because index entries are bounding boxes, not the exact shapes of the
- * indexed geometric types.  We use distance from point to MBR of index entry.
- * This is a lower bound estimate of distance from point to indexed geometric
- * type.
- */
 static float8
-gist_bbox_distance(GISTENTRY *entry, Datum query,
-				   StrategyNumber strategy, bool *recheck)
+gist_box_distance_helper(GISTENTRY *entry, Datum query,
+						 StrategyNumber strategy)
 {
 	float8		distance;
 	StrategyNumber strategyGroup = strategy / GeoStrategyNumberOffset;
 
-	/* Bounding box distance is always inexact. */
-	*recheck = true;
-
 	switch (strategyGroup)
 	{
 		case PointStrategyNumberGroup:
@@ -1500,6 +1487,42 @@ gist_bbox_distance(GISTENTRY *entry, Datum query,
 }
 
 Datum
+gist_box_distance(PG_FUNCTION_ARGS)
+{
+	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
+	Datum		query = PG_GETARG_DATUM(1);
+	StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2);
+
+	/* Oid subtype = PG_GETARG_OID(3); */
+	/* bool	   *recheck = (bool *) PG_GETARG_POINTER(4); */
+	float8		distance;
+
+	distance = gist_box_distance_helper(entry, query, strategy);
+
+	PG_RETURN_FLOAT8(distance);
+}
+
+/*
+ * The inexact GiST distance method for geometric types that store bounding
+ * boxes.
+ *
+ * Compute lossy distance from point to index entries.  The result is inexact
+ * because index entries are bounding boxes, not the exact shapes of the
+ * indexed geometric types.  We use distance from point to MBR of index entry.
+ * This is a lower bound estimate of distance from point to indexed geometric
+ * type.
+ */
+static float8
+gist_bbox_distance(GISTENTRY *entry, Datum query,
+				   StrategyNumber strategy, bool *recheck)
+{
+	/* Bounding box distance is always inexact. */
+	*recheck = true;
+
+	return gist_box_distance_helper(entry, query, strategy);
+}
+
+Datum
 gist_circle_distance(PG_FUNCTION_ARGS)
 {
 	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat
index cf63eb7..ebc38ae 100644
--- a/src/include/catalog/pg_amop.dat
+++ b/src/include/catalog/pg_amop.dat
@@ -1081,6 +1081,9 @@
   amopstrategy => '13', amopopr => '~(box,box)', amopmethod => 'gist' },
 { amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box',
   amopstrategy => '14', amopopr => '@(box,box)', amopmethod => 'gist' },
+{ amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'point',
+  amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(box,point)',
+  amopmethod => 'gist', amopsortfamily => 'btree/float_ops' },
 
 # gist point_ops
 { amopfamily => 'gist/point_ops', amoplefttype => 'point',
diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat
index 020b741..5567b7e 100644
--- a/src/include/catalog/pg_amproc.dat
+++ b/src/include/catalog/pg_amproc.dat
@@ -419,6 +419,8 @@
   amprocrighttype => 'box', amprocnum => '6', amproc => 'gist_box_picksplit' },
 { amprocfamily => 'gist/box_ops', amproclefttype => 'box',
   amprocrighttype => 'box', amprocnum => '7', amproc => 'gist_box_same' },
+{ amprocfamily => 'gist/box_ops', amproclefttype => 'box',
+  amprocrighttype => 'box', amprocnum => '8', amproc => 'gist_box_distance' },
 { amprocfamily => 'gist/poly_ops', amproclefttype => 'polygon',
   amprocrighttype => 'polygon', amprocnum => '1',
   amproc => 'gist_poly_consistent' },
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 54327bf..d15f94a 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7816,6 +7816,10 @@
 { oid => '2584', descr => 'GiST support',
   proname => 'gist_box_same', prorettype => 'internal',
   proargtypes => 'box box internal', prosrc => 'gist_box_same' },
+{ oid => '3998', descr => 'GiST support',
+  proname => 'gist_box_distance', prorettype => 'float8',
+  proargtypes => 'internal box int2 oid internal',
+  prosrc => 'gist_box_distance' },
 { oid => '2585', descr => 'GiST support',
   proname => 'gist_poly_consistent', prorettype => 'bool',
   proargtypes => 'internal polygon int2 oid internal',
diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out
index 0a43449..2234876 100644
--- a/src/test/regress/expected/gist.out
+++ b/src/test/regress/expected/gist.out
@@ -203,6 +203,82 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
  (6,6),(6,6)
 (21 rows)
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+                      QUERY PLAN                      
+------------------------------------------------------
+ Index Only Scan using gist_tbl_box_index on gist_tbl
+   Index Cond: (b <@ '(6,6),(5,5)'::box)
+   Order By: (b <-> '(5.2,5.91)'::point)
+(3 rows)
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+            b            
+-------------------------
+ (5.55,5.55),(5.55,5.55)
+ (5.6,5.6),(5.6,5.6)
+ (5.5,5.5),(5.5,5.5)
+ (5.65,5.65),(5.65,5.65)
+ (5.45,5.45),(5.45,5.45)
+ (5.7,5.7),(5.7,5.7)
+ (5.4,5.4),(5.4,5.4)
+ (5.75,5.75),(5.75,5.75)
+ (5.35,5.35),(5.35,5.35)
+ (5.8,5.8),(5.8,5.8)
+ (5.3,5.3),(5.3,5.3)
+ (5.85,5.85),(5.85,5.85)
+ (5.25,5.25),(5.25,5.25)
+ (5.9,5.9),(5.9,5.9)
+ (5.2,5.2),(5.2,5.2)
+ (5.95,5.95),(5.95,5.95)
+ (5.15,5.15),(5.15,5.15)
+ (6,6),(6,6)
+ (5.1,5.1),(5.1,5.1)
+ (5.05,5.05),(5.05,5.05)
+ (5,5),(5,5)
+(21 rows)
+
 drop index gist_tbl_box_index;
 -- Test that an index-only scan is not chosen, when the query involves the
 -- circle column (the circle opclass does not support index-only scans).
diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql
index 657b195..b9d398e 100644
--- a/src/test/regress/sql/gist.sql
+++ b/src/test/regress/sql/gist.sql
@@ -109,6 +109,22 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 -- execute the same
 select b from gist_tbl where b <@ box(point(5,5), point(6,6));
 
+-- Also test an index-only knn-search
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by b <-> point(5.2, 5.91);
+
+-- Check commuted case as well
+explain (costs off)
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
+select b from gist_tbl where b <@ box(point(5,5), point(6,6))
+order by point(5.2, 5.91) <-> b;
+
 drop index gist_tbl_box_index;
 
 -- Test that an index-only scan is not chosen, when the query involves the
-- 
2.7.4


--------------07EE9595454874974EAD75E7
Content-Type: text/x-patch;
 name="0003-Add-operator-box-point-to-SP-GiST-box_ops-v02.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename="0003-Add-operator-box-point-to-SP-GiST-box_ops-v02.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH 2/3] Fix WAL skipping feature
@ 2019-05-27 07:06 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Kyotaro Horiguchi @ 2019-05-27 07:06 UTC (permalink / raw)

WAL-skipping operations mixed with WAL-logged operations can lead to
database corruption after a crash. This patch changes the WAL-skipping
feature so that no data modifcation is WAL-logged at all then sync
such relations at commit.
---
 src/backend/access/heap/heapam.c         |  4 +-
 src/backend/access/heap/heapam_handler.c | 22 +----------
 src/backend/access/heap/rewriteheap.c    | 13 ++-----
 src/backend/catalog/storage.c            | 64 +++++++++++++++++++++++++-------
 src/backend/commands/cluster.c           | 24 ++++++++++++
 src/backend/commands/copy.c              | 39 ++++---------------
 src/backend/commands/createas.c          |  5 +--
 src/backend/commands/matview.c           |  4 --
 src/backend/commands/tablecmds.c         | 10 ++---
 src/backend/storage/buffer/bufmgr.c      | 33 +++++++++++-----
 src/backend/utils/cache/relcache.c       | 16 ++++++--
 src/include/access/heapam.h              |  1 -
 src/include/access/rewriteheap.h         |  2 +-
 src/include/access/tableam.h             | 41 ++------------------
 src/include/storage/bufmgr.h             |  1 +
 src/include/utils/rel.h                  | 17 ++++++++-
 16 files changed, 148 insertions(+), 148 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index d768b9b061..eca98fb063 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1941,7 +1941,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 	MarkBufferDirty(buffer);
 
 	/* XLOG stuff */
-	if (!(options & HEAP_INSERT_SKIP_WAL) && RelationNeedsWAL(relation))
+	if (RelationNeedsWAL(relation))
 	{
 		xl_heap_insert xlrec;
 		xl_heap_header xlhdr;
@@ -2124,7 +2124,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 	/* currently not needed (thus unsupported) for heap_multi_insert() */
 	AssertArg(!(options & HEAP_INSERT_NO_LOGICAL));
 
-	needwal = !(options & HEAP_INSERT_SKIP_WAL) && RelationNeedsWAL(relation);
+	needwal = RelationNeedsWAL(relation);
 	saveFreeSpace = RelationGetTargetPageFreeSpace(relation,
 												   HEAP_DEFAULT_FILLFACTOR);
 
diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c
index 09bc6fe98a..b9554f6064 100644
--- a/src/backend/access/heap/heapam_handler.c
+++ b/src/backend/access/heap/heapam_handler.c
@@ -556,18 +556,6 @@ tuple_lock_retry:
 	return result;
 }
 
-static void
-heapam_finish_bulk_insert(Relation relation, int options)
-{
-	/*
-	 * If we skipped writing WAL, then we need to sync the heap (but not
-	 * indexes since those use WAL anyway / don't go through tableam)
-	 */
-	if (options & HEAP_INSERT_SKIP_WAL)
-		heap_sync(relation);
-}
-
-
 /* ------------------------------------------------------------------------
  * DDL related callbacks for heap AM.
  * ------------------------------------------------------------------------
@@ -699,7 +687,6 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
 	IndexScanDesc indexScan;
 	TableScanDesc tableScan;
 	HeapScanDesc heapScan;
-	bool		use_wal;
 	bool		is_system_catalog;
 	Tuplesortstate *tuplesort;
 	TupleDesc	oldTupDesc = RelationGetDescr(OldHeap);
@@ -713,12 +700,6 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
 	/* Remember if it's a system catalog */
 	is_system_catalog = IsSystemRelation(OldHeap);
 
-	/*
-	 * We need to log the copied data in WAL iff WAL archiving/streaming is
-	 * enabled AND it's a WAL-logged rel.
-	 */
-	use_wal = XLogIsNeeded() && RelationNeedsWAL(NewHeap);
-
 	/* use_wal off requires smgr_targblock be initially invalid */
 	Assert(RelationGetTargetBlock(NewHeap) == InvalidBlockNumber);
 
@@ -729,7 +710,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
 
 	/* Initialize the rewrite operation */
 	rwstate = begin_heap_rewrite(OldHeap, NewHeap, OldestXmin, *xid_cutoff,
-								 *multi_cutoff, use_wal);
+								 *multi_cutoff);
 
 
 	/* Set up sorting if wanted */
@@ -2517,7 +2498,6 @@ static const TableAmRoutine heapam_methods = {
 	.tuple_delete = heapam_tuple_delete,
 	.tuple_update = heapam_tuple_update,
 	.tuple_lock = heapam_tuple_lock,
-	.finish_bulk_insert = heapam_finish_bulk_insert,
 
 	.tuple_fetch_row_version = heapam_fetch_row_version,
 	.tuple_get_latest_tid = heap_get_latest_tid,
diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c
index 72a448ad31..992d4b9880 100644
--- a/src/backend/access/heap/rewriteheap.c
+++ b/src/backend/access/heap/rewriteheap.c
@@ -144,7 +144,6 @@ typedef struct RewriteStateData
 	Page		rs_buffer;		/* page currently being built */
 	BlockNumber rs_blockno;		/* block where page will go */
 	bool		rs_buffer_valid;	/* T if any tuples in buffer */
-	bool		rs_use_wal;		/* must we WAL-log inserts? */
 	bool		rs_logical_rewrite; /* do we need to do logical rewriting */
 	TransactionId rs_oldest_xmin;	/* oldest xmin used by caller to determine
 									 * tuple visibility */
@@ -238,15 +237,13 @@ static void logical_end_heap_rewrite(RewriteState state);
  * oldest_xmin	xid used by the caller to determine which tuples are dead
  * freeze_xid	xid before which tuples will be frozen
  * min_multi	multixact before which multis will be removed
- * use_wal		should the inserts to the new heap be WAL-logged?
  *
  * Returns an opaque RewriteState, allocated in current memory context,
  * to be used in subsequent calls to the other functions.
  */
 RewriteState
 begin_heap_rewrite(Relation old_heap, Relation new_heap, TransactionId oldest_xmin,
-				   TransactionId freeze_xid, MultiXactId cutoff_multi,
-				   bool use_wal)
+				   TransactionId freeze_xid, MultiXactId cutoff_multi)
 {
 	RewriteState state;
 	MemoryContext rw_cxt;
@@ -271,7 +268,6 @@ begin_heap_rewrite(Relation old_heap, Relation new_heap, TransactionId oldest_xm
 	/* new_heap needn't be empty, just locked */
 	state->rs_blockno = RelationGetNumberOfBlocks(new_heap);
 	state->rs_buffer_valid = false;
-	state->rs_use_wal = use_wal;
 	state->rs_oldest_xmin = oldest_xmin;
 	state->rs_freeze_xid = freeze_xid;
 	state->rs_cutoff_multi = cutoff_multi;
@@ -330,7 +326,7 @@ end_heap_rewrite(RewriteState state)
 	/* Write the last page, if any */
 	if (state->rs_buffer_valid)
 	{
-		if (state->rs_use_wal)
+		if (RelationNeedsWAL(state->rs_new_rel))
 			log_newpage(&state->rs_new_rel->rd_node,
 						MAIN_FORKNUM,
 						state->rs_blockno,
@@ -654,9 +650,6 @@ raw_heap_insert(RewriteState state, HeapTuple tup)
 	{
 		int			options = HEAP_INSERT_SKIP_FSM;
 
-		if (!state->rs_use_wal)
-			options |= HEAP_INSERT_SKIP_WAL;
-
 		/*
 		 * While rewriting the heap for VACUUM FULL / CLUSTER, make sure data
 		 * for the TOAST table are not logically decoded.  The main heap is
@@ -695,7 +688,7 @@ raw_heap_insert(RewriteState state, HeapTuple tup)
 			/* Doesn't fit, so write out the existing page */
 
 			/* XLOG stuff */
-			if (state->rs_use_wal)
+			if (RelationNeedsWAL(state->rs_new_rel))
 				log_newpage(&state->rs_new_rel->rd_node,
 							MAIN_FORKNUM,
 							state->rs_blockno,
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f7fe..e4bcdc390f 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -57,7 +57,8 @@ typedef struct PendingRelDelete
 {
 	RelFileNode relnode;		/* relation that may need to be deleted */
 	BackendId	backend;		/* InvalidBackendId if not a temp rel */
-	bool		atCommit;		/* T=delete at commit; F=delete at abort */
+	bool		atCommit;		/* T=work at commit; F=work at abort */
+	bool		dosync;			/* T=work is sync; F=work is delete */
 	int			nestLevel;		/* xact nesting level of request */
 	struct PendingRelDelete *next;	/* linked-list link */
 } PendingRelDelete;
@@ -114,10 +115,29 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
 	pending->relnode = rnode;
 	pending->backend = backend;
 	pending->atCommit = false;	/* delete if abort */
+	pending->dosync = false;
 	pending->nestLevel = GetCurrentTransactionNestLevel();
 	pending->next = pendingDeletes;
 	pendingDeletes = pending;
 
+	/*
+	 * We are going to skip WAL-logging for storage of persistent relations
+	 * created in the current transaction when wal_level = minimal. The
+	 * relation needs to be synced at commit.
+	 */
+	if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
+	{
+		pending = (PendingRelDelete *)
+			MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+		pending->relnode = rnode;
+		pending->backend = backend;
+		pending->atCommit = true;
+		pending->dosync = true;
+		pending->nestLevel = GetCurrentTransactionNestLevel();
+		pending->next = pendingDeletes;
+		pendingDeletes = pending;
+	}
+
 	return srel;
 }
 
@@ -155,6 +175,7 @@ RelationDropStorage(Relation rel)
 	pending->relnode = rel->rd_node;
 	pending->backend = rel->rd_backend;
 	pending->atCommit = true;	/* delete if commit */
+	pending->dosync = false;
 	pending->nestLevel = GetCurrentTransactionNestLevel();
 	pending->next = pendingDeletes;
 	pendingDeletes = pending;
@@ -428,21 +449,34 @@ smgrDoPendingDeletes(bool isCommit)
 			{
 				SMgrRelation srel;
 
-				srel = smgropen(pending->relnode, pending->backend);
-
-				/* allocate the initial array, or extend it, if needed */
-				if (maxrels == 0)
+				if (pending->dosync)
 				{
-					maxrels = 8;
-					srels = palloc(sizeof(SMgrRelation) * maxrels);
+					/* Perform pending sync of WAL-skipped relation */
+					FlushRelationBuffersWithoutRelcache(pending->relnode,
+														false);
+					srel = smgropen(pending->relnode, pending->backend);
+					smgrimmedsync(srel, MAIN_FORKNUM);
+					smgrclose(srel);
 				}
-				else if (maxrels <= nrels)
+				else
 				{
-					maxrels *= 2;
-					srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
-				}
+					/* Collect pending deletions */
+					srel = smgropen(pending->relnode, pending->backend);
 
-				srels[nrels++] = srel;
+					/* allocate the initial array, or extend it, if needed */
+					if (maxrels == 0)
+					{
+						maxrels = 8;
+						srels = palloc(sizeof(SMgrRelation) * maxrels);
+					}
+					else if (maxrels <= nrels)
+					{
+						maxrels *= 2;
+						srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
+					}
+
+					srels[nrels++] = srel;
+				}
 			}
 			/* must explicitly free the list entry */
 			pfree(pending);
@@ -489,8 +523,9 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
 	nrels = 0;
 	for (pending = pendingDeletes; pending != NULL; pending = pending->next)
 	{
+		/* Pending syncs are excluded */
 		if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
-			&& pending->backend == InvalidBackendId)
+			&& pending->backend == InvalidBackendId && !pending->dosync)
 			nrels++;
 	}
 	if (nrels == 0)
@@ -502,8 +537,9 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
 	*ptr = rptr;
 	for (pending = pendingDeletes; pending != NULL; pending = pending->next)
 	{
+		/* Pending syncs are excluded */
 		if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
-			&& pending->backend == InvalidBackendId)
+			&& pending->backend == InvalidBackendId && !pending->dosync)
 		{
 			*rptr = pending->relnode;
 			rptr++;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index ebaec4f8dd..6fc9d7d64e 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -1034,12 +1034,36 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class,
 
 	if (OidIsValid(relfilenode1) && OidIsValid(relfilenode2))
 	{
+		Relation rel1;
+		Relation rel2;
+
 		/*
 		 * Normal non-mapped relations: swap relfilenodes, reltablespaces,
 		 * relpersistence
 		 */
 		Assert(!target_is_pg_class);
 
+		/* Update creation subid hints of relcache */
+		rel1 = relation_open(r1, ExclusiveLock);
+		rel2 = relation_open(r2, ExclusiveLock);
+
+		/*
+		 * New relation's relfilenode is created in the current transaction
+		 * and used as old ralation's new relfilenode. So its
+		 * newRelfilenodeSubid as new relation's createSubid. We don't fix
+		 * rel2 since it would be deleted soon.
+		 */
+		Assert(rel2->rd_createSubid != InvalidSubTransactionId);
+		rel1->rd_newRelfilenodeSubid = rel2->rd_createSubid;
+
+		/* record the first relfilenode change in the current transaction */
+		if (rel1->rd_firstRelfilenodeSubid == InvalidSubTransactionId)
+			rel1->rd_firstRelfilenodeSubid = GetCurrentSubTransactionId();
+
+		relation_close(rel1, ExclusiveLock);
+		relation_close(rel2, ExclusiveLock);
+
+		/* swap relfilenodes, reltablespaces, relpersistence */
 		swaptemp = relform1->relfilenode;
 		relform1->relfilenode = relform2->relfilenode;
 		relform2->relfilenode = swaptemp;
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 4f04d122c3..f02efd59fc 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2535,9 +2535,6 @@ CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo,
 	for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++)
 		ExecDropSingleTupleTableSlot(buffer->slots[i]);
 
-	table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc,
-							 miinfo->ti_options);
-
 	pfree(buffer);
 }
 
@@ -2726,28 +2723,9 @@ CopyFrom(CopyState cstate)
 	 * If it does commit, we'll have done the table_finish_bulk_insert() at
 	 * the bottom of this routine first.
 	 *
-	 * As mentioned in comments in utils/rel.h, the in-same-transaction test
-	 * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
-	 * can be cleared before the end of the transaction. The exact case is
-	 * when a relation sets a new relfilenode twice in same transaction, yet
-	 * the second one fails in an aborted subtransaction, e.g.
-	 *
-	 * BEGIN;
-	 * TRUNCATE t;
-	 * SAVEPOINT save;
-	 * TRUNCATE t;
-	 * ROLLBACK TO save;
-	 * COPY ...
-	 *
-	 * Also, if the target file is new-in-transaction, we assume that checking
-	 * FSM for free space is a waste of time, even if we must use WAL because
-	 * of archiving.  This could possibly be wrong, but it's unlikely.
-	 *
-	 * The comments for table_tuple_insert and RelationGetBufferForTuple
-	 * specify that skipping WAL logging is only safe if we ensure that our
-	 * tuples do not go into pages containing tuples from any other
-	 * transactions --- but this must be the case if we have a new table or
-	 * new relfilenode, so we need no additional work to enforce that.
+	 * If the target file is new-in-transaction, we assume that checking FSM
+	 * for free space is a waste of time, even if we must use WAL because of
+	 * archiving.  This could possibly be wrong, but it's unlikely.
 	 *
 	 * We currently don't support this optimization if the COPY target is a
 	 * partitioned table as we currently only lazily initialize partition
@@ -2763,15 +2741,14 @@ CopyFrom(CopyState cstate)
 	 * are not supported as per the description above.
 	 *----------
 	 */
-	/* createSubid is creation check, newRelfilenodeSubid is truncation check */
+	/*
+	 * createSubid is creation check, firstRelfilenodeSubid is truncation and
+	 * cluster check. Partitioned table doesn't have storage.
+	 */
 	if (RELKIND_HAS_STORAGE(cstate->rel->rd_rel->relkind) &&
 		(cstate->rel->rd_createSubid != InvalidSubTransactionId ||
-		 cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId))
-	{
+		 cstate->rel->rd_firstRelfilenodeSubid != InvalidSubTransactionId))
 		ti_options |= TABLE_INSERT_SKIP_FSM;
-		if (!XLogIsNeeded())
-			ti_options |= TABLE_INSERT_SKIP_WAL;
-	}
 
 	/*
 	 * Optimize if new relfilenode was created in this subxact or one of its
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 4c1d909d38..39ebd73691 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -558,8 +558,7 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	 * We can skip WAL-logging the insertions, unless PITR or streaming
 	 * replication is in use. We can skip the FSM in any case.
 	 */
-	myState->ti_options = TABLE_INSERT_SKIP_FSM |
-		(XLogIsNeeded() ? 0 : TABLE_INSERT_SKIP_WAL);
+	myState->ti_options = TABLE_INSERT_SKIP_FSM;
 	myState->bistate = GetBulkInsertState();
 
 	/* Not using WAL requires smgr_targblock be initially invalid */
@@ -604,8 +603,6 @@ intorel_shutdown(DestReceiver *self)
 
 	FreeBulkInsertState(myState->bistate);
 
-	table_finish_bulk_insert(myState->rel, myState->ti_options);
-
 	/* close rel, but keep lock until commit */
 	table_close(myState->rel, NoLock);
 	myState->rel = NULL;
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 537d0e8cef..1c854dcebf 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -463,8 +463,6 @@ transientrel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	 * replication is in use. We can skip the FSM in any case.
 	 */
 	myState->ti_options = TABLE_INSERT_SKIP_FSM | TABLE_INSERT_FROZEN;
-	if (!XLogIsNeeded())
-		myState->ti_options |= TABLE_INSERT_SKIP_WAL;
 	myState->bistate = GetBulkInsertState();
 
 	/* Not using WAL requires smgr_targblock be initially invalid */
@@ -509,8 +507,6 @@ transientrel_shutdown(DestReceiver *self)
 
 	FreeBulkInsertState(myState->bistate);
 
-	table_finish_bulk_insert(myState->transientrel, myState->ti_options);
-
 	/* close transientrel, but keep lock until commit */
 	table_close(myState->transientrel, NoLock);
 	myState->transientrel = NULL;
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 0f1a9f0e54..ac7336ef58 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -4761,9 +4761,9 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode)
 
 	/*
 	 * Prepare a BulkInsertState and options for table_tuple_insert. Because
-	 * we're building a new heap, we can skip WAL-logging and fsync it to disk
-	 * at the end instead (unless WAL-logging is required for archiving or
-	 * streaming replication). The FSM is empty too, so don't bother using it.
+	 * we're building a new heap, the underlying table AM can skip WAL-logging
+	 * and fsync the relation to disk at the end of the current transaction
+	 * instead. The FSM is empty too, so don't bother using it.
 	 */
 	if (newrel)
 	{
@@ -4771,8 +4771,6 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode)
 		bistate = GetBulkInsertState();
 
 		ti_options = TABLE_INSERT_SKIP_FSM;
-		if (!XLogIsNeeded())
-			ti_options |= TABLE_INSERT_SKIP_WAL;
 	}
 	else
 	{
@@ -5057,8 +5055,6 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode)
 	{
 		FreeBulkInsertState(bistate);
 
-		table_finish_bulk_insert(newrel, ti_options);
-
 		table_close(newrel, NoLock);
 	}
 }
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b590..280fdf8080 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -171,6 +171,7 @@ static HTAB *PrivateRefCountHash = NULL;
 static int32 PrivateRefCountOverflowed = 0;
 static uint32 PrivateRefCountClock = 0;
 static PrivateRefCountEntry *ReservedRefCountEntry = NULL;
+static void FlushRelationBuffers_common(SMgrRelation smgr, bool islocal);
 
 static void ReservePrivateRefCountEntry(void);
 static PrivateRefCountEntry *NewPrivateRefCountEntry(Buffer buffer);
@@ -3190,20 +3191,32 @@ PrintPinnedBufs(void)
 void
 FlushRelationBuffers(Relation rel)
 {
-	int			i;
-	BufferDesc *bufHdr;
-
-	/* Open rel at the smgr level if not already done */
 	RelationOpenSmgr(rel);
 
-	if (RelationUsesLocalBuffers(rel))
+	FlushRelationBuffers_common(rel->rd_smgr, RelationUsesLocalBuffers(rel));
+}
+
+void
+FlushRelationBuffersWithoutRelcache(RelFileNode rnode, bool islocal)
+{
+	FlushRelationBuffers_common(smgropen(rnode, InvalidBackendId), islocal);
+}
+
+static void
+FlushRelationBuffers_common(SMgrRelation smgr, bool islocal)
+{
+	RelFileNode rnode = smgr->smgr_rnode.node;
+	int i;
+	BufferDesc *bufHdr;
+
+	if (islocal)
 	{
 		for (i = 0; i < NLocBuffer; i++)
 		{
 			uint32		buf_state;
 
 			bufHdr = GetLocalBufferDescriptor(i);
-			if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) &&
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode) &&
 				((buf_state = pg_atomic_read_u32(&bufHdr->state)) &
 				 (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
 			{
@@ -3220,7 +3233,7 @@ FlushRelationBuffers(Relation rel)
 
 				PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
 
-				smgrwrite(rel->rd_smgr,
+				smgrwrite(smgr,
 						  bufHdr->tag.forkNum,
 						  bufHdr->tag.blockNum,
 						  localpage,
@@ -3250,18 +3263,18 @@ FlushRelationBuffers(Relation rel)
 		 * As in DropRelFileNodeBuffers, an unlocked precheck should be safe
 		 * and saves some cycles.
 		 */
-		if (!RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node))
+		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode))
 			continue;
 
 		ReservePrivateRefCountEntry();
 
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) &&
+		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode) &&
 			(buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
 		{
 			PinBuffer_Locked(bufHdr);
 			LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
-			FlushBuffer(bufHdr, rel->rd_smgr);
+			FlushBuffer(bufHdr, smgr);
 			LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
 			UnpinBuffer(bufHdr, true);
 		}
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 2b992d7832..812bfadb40 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -2661,7 +2661,7 @@ static void
 RelationFlushRelation(Relation relation)
 {
 	if (relation->rd_createSubid != InvalidSubTransactionId ||
-		relation->rd_newRelfilenodeSubid != InvalidSubTransactionId)
+		relation->rd_firstRelfilenodeSubid != InvalidSubTransactionId)
 	{
 		/*
 		 * New relcache entries are always rebuilt, not flushed; else we'd
@@ -2801,7 +2801,7 @@ RelationCacheInvalidate(void)
 		 * pending invalidations.
 		 */
 		if (relation->rd_createSubid != InvalidSubTransactionId ||
-			relation->rd_newRelfilenodeSubid != InvalidSubTransactionId)
+			relation->rd_firstRelfilenodeSubid != InvalidSubTransactionId)
 			continue;
 
 		relcacheInvalsReceived++;
@@ -3058,6 +3058,7 @@ AtEOXact_cleanup(Relation relation, bool isCommit)
 	 * Likewise, reset the hint about the relfilenode being new.
 	 */
 	relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
+	relation->rd_firstRelfilenodeSubid = InvalidSubTransactionId;
 }
 
 /*
@@ -3149,7 +3150,7 @@ AtEOSubXact_cleanup(Relation relation, bool isCommit,
 	}
 
 	/*
-	 * Likewise, update or drop any new-relfilenode-in-subtransaction hint.
+	 * Likewise, update or drop any new-relfilenode-in-subtransaction hints.
 	 */
 	if (relation->rd_newRelfilenodeSubid == mySubid)
 	{
@@ -3158,6 +3159,15 @@ AtEOSubXact_cleanup(Relation relation, bool isCommit,
 		else
 			relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
 	}
+
+
+	if (relation->rd_firstRelfilenodeSubid == mySubid)
+	{
+		if (isCommit)
+			relation->rd_firstRelfilenodeSubid = parentSubid;
+		else
+			relation->rd_firstRelfilenodeSubid = InvalidSubTransactionId;
+	}
 }
 
 
diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h
index 858bcb6bc9..80c2e1bafc 100644
--- a/src/include/access/heapam.h
+++ b/src/include/access/heapam.h
@@ -29,7 +29,6 @@
 
 
 /* "options" flag bits for heap_insert */
-#define HEAP_INSERT_SKIP_WAL	TABLE_INSERT_SKIP_WAL
 #define HEAP_INSERT_SKIP_FSM	TABLE_INSERT_SKIP_FSM
 #define HEAP_INSERT_FROZEN		TABLE_INSERT_FROZEN
 #define HEAP_INSERT_NO_LOGICAL	TABLE_INSERT_NO_LOGICAL
diff --git a/src/include/access/rewriteheap.h b/src/include/access/rewriteheap.h
index 8056253916..7f9736e294 100644
--- a/src/include/access/rewriteheap.h
+++ b/src/include/access/rewriteheap.h
@@ -23,7 +23,7 @@ typedef struct RewriteStateData *RewriteState;
 
 extern RewriteState begin_heap_rewrite(Relation OldHeap, Relation NewHeap,
 									   TransactionId OldestXmin, TransactionId FreezeXid,
-									   MultiXactId MultiXactCutoff, bool use_wal);
+									   MultiXactId MultiXactCutoff);
 extern void end_heap_rewrite(RewriteState state);
 extern void rewrite_heap_tuple(RewriteState state, HeapTuple oldTuple,
 							   HeapTuple newTuple);
diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h
index c2b0481e7e..ac0e981acb 100644
--- a/src/include/access/tableam.h
+++ b/src/include/access/tableam.h
@@ -407,22 +407,6 @@ typedef struct TableAmRoutine
 							   uint8 flags,
 							   TM_FailureData *tmfd);
 
-	/*
-	 * Perform operations necessary to complete insertions made via
-	 * tuple_insert and multi_insert with a BulkInsertState specified. This
-	 * may for example be used to flush the relation, when the
-	 * TABLE_INSERT_SKIP_WAL option was used.
-	 *
-	 * Typically callers of tuple_insert and multi_insert will just pass all
-	 * the flags that apply to them, and each AM has to decide which of them
-	 * make sense for it, and then only take actions in finish_bulk_insert for
-	 * those flags, and ignore others.
-	 *
-	 * Optional callback.
-	 */
-	void		(*finish_bulk_insert) (Relation rel, int options);
-
-
 	/* ------------------------------------------------------------------------
 	 * DDL related functionality.
 	 * ------------------------------------------------------------------------
@@ -1088,10 +1072,6 @@ table_compute_xid_horizon_for_tuples(Relation rel,
  * behaviour of the AM. Several options might be ignored by AMs not supporting
  * them.
  *
- * If the TABLE_INSERT_SKIP_WAL option is specified, the new tuple doesn't
- * need to be logged to WAL, even for a non-temp relation. It is the AMs
- * choice whether this optimization is supported.
- *
  * If the TABLE_INSERT_SKIP_FSM option is specified, AMs are free to not reuse
  * free space in the relation. This can save some cycles when we know the
  * relation is new and doesn't contain useful amounts of free space.  It's
@@ -1111,10 +1091,8 @@ table_compute_xid_horizon_for_tuples(Relation rel,
  * Note that most of these options will be applied when inserting into the
  * heap's TOAST table, too, if the tuple requires any out-of-line data.
  *
- *
  * The BulkInsertState object (if any; bistate can be NULL for default
- * behavior) is also just passed through to RelationGetBufferForTuple. If
- * `bistate` is provided, table_finish_bulk_insert() needs to be called.
+ * behavior) is also just passed through to RelationGetBufferForTuple.
  *
  * On return the slot's tts_tid and tts_tableOid are updated to reflect the
  * insertion. But note that any toasting of fields within the slot is NOT
@@ -1249,6 +1227,8 @@ table_tuple_delete(Relation rel, ItemPointer tid, CommandId cid,
  * update was done.  However, any TOAST changes in the new tuple's
  * data are not reflected into *newtup.
  *
+ * See table_insert about skipping WAL-logging feature.
+ *
  * In the failure cases, the routine fills *tmfd with the tuple's t_ctid,
  * t_xmax, and, if possible, t_cmax.  See comments for struct TM_FailureData
  * for additional info.
@@ -1309,21 +1289,6 @@ table_tuple_lock(Relation rel, ItemPointer tid, Snapshot snapshot,
 									   flags, tmfd);
 }
 
-/*
- * Perform operations necessary to complete insertions made via
- * tuple_insert and multi_insert with a BulkInsertState specified. This
- * e.g. may e.g. used to flush the relation when inserting with
- * TABLE_INSERT_SKIP_WAL specified.
- */
-static inline void
-table_finish_bulk_insert(Relation rel, int options)
-{
-	/* optional callback */
-	if (rel->rd_tableam && rel->rd_tableam->finish_bulk_insert)
-		rel->rd_tableam->finish_bulk_insert(rel, options);
-}
-
-
 /* ------------------------------------------------------------------------
  * DDL related functionality.
  * ------------------------------------------------------------------------
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7ef1..ace5f5a2ae 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -189,6 +189,7 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 												   ForkNumber forkNum);
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
+extern void FlushRelationBuffersWithoutRelcache(RelFileNode rnode, bool islocal);
 extern void FlushDatabaseBuffers(Oid dbid);
 extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
 								   ForkNumber forkNum, BlockNumber firstDelBlock);
diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h
index d35b4a5061..5cbb5a7b27 100644
--- a/src/include/utils/rel.h
+++ b/src/include/utils/rel.h
@@ -76,10 +76,17 @@ typedef struct RelationData
 	 * transaction, with one of them occurring in a subsequently aborted
 	 * subtransaction, e.g. BEGIN; TRUNCATE t; SAVEPOINT save; TRUNCATE t;
 	 * ROLLBACK TO save; -- rd_newRelfilenode is now forgotten
+	 * rd_firstRelfilenodeSubid is the ID of the hightest subtransaction the
+	 * relfilenode change has took place first in the current
+	 * transaction. This won't be forgotten as newRelfilenodeSubid is. A valid
+	 * OID means that the currently active relfilenode is transaction-local
+	 * and no-need for WAL-logging.
 	 */
 	SubTransactionId rd_createSubid;	/* rel was created in current xact */
 	SubTransactionId rd_newRelfilenodeSubid;	/* new relfilenode assigned in
 												 * current xact */
+	SubTransactionId rd_firstRelfilenodeSubid;	/* new relfilenode assigned
+												 * first in current xact */
 
 	Form_pg_class rd_rel;		/* RELATION tuple */
 	TupleDesc	rd_att;			/* tuple descriptor */
@@ -512,9 +519,15 @@ typedef struct ViewOptions
 /*
  * RelationNeedsWAL
  *		True if relation needs WAL.
+ *
+ * Returns false if wal_level = minimal and this relation is created or
+ * truncated in the current transaction.
  */
-#define RelationNeedsWAL(relation) \
-	((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT)
+#define RelationNeedsWAL(relation)										\
+	((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT &&	\
+	 (XLogIsNeeded() ||													\
+	  (relation->rd_createSubid == InvalidSubTransactionId &&			\
+	   relation->rd_firstRelfilenodeSubid == InvalidSubTransactionId)))
 
 /*
  * RelationUsesLocalBuffers
-- 
2.16.3


----Next_Part(Fri_Jul_12_17_37_25_2019_721)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v17-0003-Rename-smgrDoPendingDeletes-to-smgrDoPendingOperatio.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v1 4/7] Row pattern recognition patch (executor).
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 223 +++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 274 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  12 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 517 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..ae997dff86 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2762,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2849,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2900,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3242,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3582,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3784,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3821,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..63b225271c 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -39,7 +42,9 @@ typedef struct
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +718,270 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+		PG_RETURN_DATUM(result);
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..bc95c5fe9b 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,13 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('*', '+' or '?'. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2562,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v1-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v1 4/7] Row pattern recognition patch (executor).
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 223 +++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 274 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  12 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 517 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..ae997dff86 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2762,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2849,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2900,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3242,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3582,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3784,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3821,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..63b225271c 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -39,7 +42,9 @@ typedef struct
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +718,270 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+		PG_RETURN_DATUM(result);
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..bc95c5fe9b 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,13 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('*', '+' or '?'. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2562,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v1-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v1 4/7] Row pattern recognition patch (executor).
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 223 +++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 274 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  12 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 517 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..ae997dff86 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2762,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2849,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2900,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3242,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3582,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3784,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3821,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..63b225271c 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -39,7 +42,9 @@ typedef struct
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +718,270 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+		PG_RETURN_DATUM(result);
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..bc95c5fe9b 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,13 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('*', '+' or '?'. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2562,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v1-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v1 4/7] Row pattern recognition patch (executor).
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 223 +++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 274 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  12 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 517 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..ae997dff86 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2762,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2849,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2900,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3242,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3582,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3784,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3821,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..63b225271c 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -39,7 +42,9 @@ typedef struct
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +718,270 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+		PG_RETURN_DATUM(result);
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..bc95c5fe9b 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,13 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('*', '+' or '?'. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2562,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v1-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v1 4/7] Row pattern recognition patch (executor).
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 223 +++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 274 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  12 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 517 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..ae997dff86 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2762,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2849,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2900,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3242,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3582,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3784,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3821,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..63b225271c 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -39,7 +42,9 @@ typedef struct
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +718,270 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+		PG_RETURN_DATUM(result);
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..bc95c5fe9b 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,13 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('*', '+' or '?'. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2562,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v1-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v2 4/7] Row pattern recognition patch (executor).
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 225 +++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 302 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  13 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 548 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..bef2bc62b2 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,71 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2764,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2851,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2902,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3244,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3584,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3786,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3823,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..74ef11ce55 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,10 +39,21 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +727,289 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+	SkipContext	*context = NULL;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+	{
+		context = (SkipContext *) WinGetPartitionLocalMemory(winobj, sizeof(SkipContext));
+		if (curr_pos < context->pos)
+		{
+			elog(DEBUG1, "skip this row: curr_pos: " INT64_FORMAT "context->pos: " INT64_FORMAT,
+				 curr_pos, context->pos);
+			PG_RETURN_NULL();
+		}
+	}
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+	{
+		if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+		{
+			context->pos += relpos;
+			elog(DEBUG1, "context->pos: " INT64_FORMAT, context->pos);
+		}
+		PG_RETURN_DATUM(result);
+	}
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..1643eaa6f1 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v2-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v2 4/7] Row pattern recognition patch (executor).
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 225 +++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 302 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  13 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 548 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..bef2bc62b2 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,71 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2764,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2851,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2902,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3244,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3584,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3786,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3823,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..74ef11ce55 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,10 +39,21 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +727,289 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+	SkipContext	*context = NULL;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+	{
+		context = (SkipContext *) WinGetPartitionLocalMemory(winobj, sizeof(SkipContext));
+		if (curr_pos < context->pos)
+		{
+			elog(DEBUG1, "skip this row: curr_pos: " INT64_FORMAT "context->pos: " INT64_FORMAT,
+				 curr_pos, context->pos);
+			PG_RETURN_NULL();
+		}
+	}
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+	{
+		if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+		{
+			context->pos += relpos;
+			elog(DEBUG1, "context->pos: " INT64_FORMAT, context->pos);
+		}
+		PG_RETURN_DATUM(result);
+	}
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..1643eaa6f1 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v2-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v2 4/7] Row pattern recognition patch (executor).
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 225 +++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 302 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  13 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 548 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..bef2bc62b2 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,71 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2764,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2851,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2902,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3244,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3584,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3786,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3823,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..74ef11ce55 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,10 +39,21 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +727,289 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+	SkipContext	*context = NULL;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+	{
+		context = (SkipContext *) WinGetPartitionLocalMemory(winobj, sizeof(SkipContext));
+		if (curr_pos < context->pos)
+		{
+			elog(DEBUG1, "skip this row: curr_pos: " INT64_FORMAT "context->pos: " INT64_FORMAT,
+				 curr_pos, context->pos);
+			PG_RETURN_NULL();
+		}
+	}
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+	{
+		if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+		{
+			context->pos += relpos;
+			elog(DEBUG1, "context->pos: " INT64_FORMAT, context->pos);
+		}
+		PG_RETURN_DATUM(result);
+	}
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..1643eaa6f1 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v2-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v2 4/7] Row pattern recognition patch (executor).
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 225 +++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 302 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  13 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 548 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..bef2bc62b2 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,71 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2764,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2851,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2902,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3244,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3584,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3786,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3823,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..74ef11ce55 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,10 +39,21 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +727,289 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+	SkipContext	*context = NULL;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+	{
+		context = (SkipContext *) WinGetPartitionLocalMemory(winobj, sizeof(SkipContext));
+		if (curr_pos < context->pos)
+		{
+			elog(DEBUG1, "skip this row: curr_pos: " INT64_FORMAT "context->pos: " INT64_FORMAT,
+				 curr_pos, context->pos);
+			PG_RETURN_NULL();
+		}
+	}
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+	{
+		if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+		{
+			context->pos += relpos;
+			elog(DEBUG1, "context->pos: " INT64_FORMAT, context->pos);
+		}
+		PG_RETURN_DATUM(result);
+	}
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..1643eaa6f1 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v2-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v2 4/7] Row pattern recognition patch (executor).
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 225 +++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  | 302 ++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.dat      |   9 +
 src/include/nodes/execnodes.h        |  13 ++
 src/include/windowapi.h              |   9 +
 5 files changed, 548 insertions(+), 10 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..bef2bc62b2 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -48,6 +48,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +160,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -195,9 +204,9 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
 
 /*
  * initialize_windowaggregate
@@ -2388,6 +2397,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2498,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2692,71 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+			if (func->winfnoid != F_RPR)
+				continue;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "RPR must have 1 argument but function %d has %d args", func->winfnoid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "RPR's arg is not Var");
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2764,76 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2851,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2902,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3244,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3420,14 +3584,53 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3583,15 +3786,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3823,9 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..74ef11ce55 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,10 +39,21 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
-
+static bool get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos);
+static int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+							int relpos, char *vname, char *quantifier, bool *result);
 
 /*
  * utility routine for *_rank functions.
@@ -713,3 +727,289 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * rpr
+ * allow to use "Row pattern recognition: WINDOW clause" (SQL:2016 R020) in
+ * the target list.
+ * Usage: SELECT rpr(colname) OVER (..)
+ * where colname is defined in PATTERN clause.
+ */
+Datum
+window_rpr(PG_FUNCTION_ARGS)
+{
+#define	MAX_PATTERNS	16	/* max variables in PATTERN clause */
+
+	WindowObject winobj = PG_WINDOW_OBJECT();
+	WindowAggState	*winstate = WinGetAggState(winobj);
+	Datum		result;
+	bool		expression_result;
+	bool		isnull;
+	int			relpos;
+	int64		curr_pos, markpos;
+	ListCell	*lc, *lc1;
+	SkipContext	*context = NULL;
+
+	curr_pos = WinGetCurrentPosition(winobj);
+	elog(DEBUG1, "rpr is called. row: " INT64_FORMAT, curr_pos);
+
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+	{
+		context = (SkipContext *) WinGetPartitionLocalMemory(winobj, sizeof(SkipContext));
+		if (curr_pos < context->pos)
+		{
+			elog(DEBUG1, "skip this row: curr_pos: " INT64_FORMAT "context->pos: " INT64_FORMAT,
+				 curr_pos, context->pos);
+			PG_RETURN_NULL();
+		}
+	}
+
+	/*
+	 * Evaluate PATTERN until one of expressions is not true or out of frame.
+	 */
+	relpos = 0;
+
+	forboth(lc, winstate->patternVariableList, lc1, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc));
+		char	*quantifier = strVal(lfirst(lc1));
+
+		elog(DEBUG1, "relpos: %d pattern vname: %s quantifier: %s", relpos, vname, quantifier);
+
+		/* evaluate row pattern against current row */
+		relpos = evaluate_pattern(winobj, winstate, relpos, vname, quantifier, &expression_result);
+
+		/*
+		 * If the expression did not match, we are done.
+		 */
+		if (!expression_result)
+			break;
+
+		/* out of frame? */
+		if (relpos < 0)
+			break;
+
+		/* count up relative row position */
+		relpos++;
+	}
+
+	elog(DEBUG1, "relpos: %d", relpos);
+
+	/*
+	 * If current row satified the pattern, return argument expression.
+	 */
+	if (expression_result)
+	{
+		result = WinGetFuncArgInFrame(winobj, 0,
+									  0, WINDOW_SEEK_HEAD, false,
+									  &isnull, NULL);
+	}
+
+	/*
+	 * At this point we can set mark down to current pos -2.
+	 */
+	markpos = curr_pos -2;
+	elog(DEBUG1, "markpos: " INT64_FORMAT, markpos);
+	if (markpos >= 0)
+		WinSetMarkPosition(winobj, markpos);
+
+	if (expression_result)
+	{
+		if (winstate->rpSkipTo == ST_PAST_LAST_ROW)
+		{
+			context->pos += relpos;
+			elog(DEBUG1, "context->pos: " INT64_FORMAT, context->pos);
+		}
+		PG_RETURN_DATUM(result);
+	}
+
+	PG_RETURN_NULL();
+}
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match relative row position
+ * -1: current row is out of frame
+ */
+static
+int evaluate_pattern(WindowObject winobj, WindowAggState *winstate,
+					 int relpos, char *vname, char *quantifier, bool *result)
+{
+	ExprContext	*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell	*lc1, *lc2;
+	ExprState	*pat;
+	Datum		eval_result;
+	int			sts;
+	bool		out_of_frame = false;
+	bool		isnull;
+	StringInfo	encoded_str = makeStringInfo();
+	char		pattern_str[128];
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+		bool	second_try_match = false;
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		for (;;)
+		{
+			if (!get_slots(winobj, winstate, relpos))
+			{
+				out_of_frame = true;
+				break;	/* current row is out of frame */
+			}
+
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: %d", vname, relpos);
+				break;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: %d", vname, relpos);
+					break;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: %d", vname, relpos);
+					appendStringInfoChar(encoded_str, vname[0]);
+
+					/* If quantifier is "+", we need to look for more matching row */
+					if (quantifier && !strcmp(quantifier, "+"))
+					{
+						/* remember that we want to try another row */
+						second_try_match = true;
+						relpos++;
+					}
+					else
+						break;
+				}
+			}
+		}
+		if (second_try_match)
+			relpos--;
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+
+		/* build regular expression */
+		snprintf(pattern_str, sizeof(pattern_str), "%c%s", vname[0], quantifier);
+
+		/*
+		 * Do regular expression matching against sequence of rows satisfying
+		 * the expression using regexp_instr().
+		 */
+		sts = DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+													PointerGetDatum(cstring_to_text(encoded_str->data)),
+													PointerGetDatum(cstring_to_text(pattern_str))));
+		elog(DEBUG1, "regexp_instr returned: %d. str: %s regexp: %s",
+			 sts, encoded_str->data, pattern_str);
+		*result = (sts > 0)? true : false;
+	}
+	return relpos;
+}
+
+/*
+ * Get current, previous and next tuple.
+ * Returns true if still within frame.
+ */
+static bool
+get_slots(WindowObject winobj, WindowAggState *winstate, int current_pos)
+{
+	TupleTableSlot *slot;
+	bool	isnull, isout;
+	int		sts;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* for current row */
+	slot = winstate->temp_slot_1;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "current row is out of frame");
+		econtext->ecxt_scantuple = winstate->null_slot;
+		return false;
+	}
+	else
+		econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		sts = WinGetSlotInFrame(winobj, slot,
+								current_pos - 1, WINDOW_SEEK_HEAD, false,
+								&isnull, &isout);
+		if (sts < 0)
+		{
+			elog(DEBUG1, "previous row out of frame at: %d", current_pos);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		econtext->ecxt_outertuple = slot;
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	sts = WinGetSlotInFrame(winobj, slot,
+							current_pos + 1, WINDOW_SEEK_HEAD, false,
+							&isnull, &isout);
+	if (sts < 0)
+	{
+		elog(DEBUG1, "next row out of frame at: %d", current_pos);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+		econtext->ecxt_innertuple = slot;
+
+	return true;
+}
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..e3a9e0ffeb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,15 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'row pattern recognition in window',
+  proname => 'rpr', prokind => 'w', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_rpr' },
+{ oid => '6123', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6124', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..1643eaa6f1 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,11 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..a0facf38fe 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,16 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v2-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v3 4/7] Row pattern recognition patch (executor).
@ 2023-07-26 10:49 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-07-26 10:49 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 701 ++++++++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  |  38 +-
 src/include/catalog/pg_proc.dat      |   6 +
 src/include/nodes/execnodes.h        |  18 +
 src/include/windowapi.h              |   8 +
 5 files changed, 758 insertions(+), 13 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..0586bf57d6 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -36,6 +36,7 @@
 #include "access/htup_details.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_aggregate.h"
+#include "catalog/pg_collation_d.h"
 #include "catalog/pg_proc.h"
 #include "executor/executor.h"
 #include "executor/nodeWindowAgg.h"
@@ -48,6 +49,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +161,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -182,8 +192,9 @@ static void begin_partition(WindowAggState *winstate);
 static void spool_tuples(WindowAggState *winstate, int64 pos);
 static void release_partition(WindowAggState *winstate);
 
-static int	row_is_in_frame(WindowAggState *winstate, int64 pos,
+static int  row_is_in_frame(WindowAggState *winstate, int64 pos,
 							TupleTableSlot *slot);
+
 static void update_frameheadpos(WindowAggState *winstate);
 static void update_frametailpos(WindowAggState *winstate);
 static void update_grouptailpos(WindowAggState *winstate);
@@ -195,9 +206,19 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
+static int row_is_in_reduced_frame(WindowObject winobj, int64 pos);
+
+static int64 evaluate_pattern(WindowObject winobj, int64 current_pos, 
+							  char *vname, StringInfo encoded_str, bool *result);
+
+static bool get_slots(WindowObject winobj, int64 current_pos);
+
+static int search_str_set(char *pattern, StringInfo *str_set, int set_size);
+static void search_str_set_recurse(char *pattern, StringInfo *str_set, int set_size, int set_index,
+								   char *encoded_str, int *resultlen);
 
 /*
  * initialize_windowaggregate
@@ -673,6 +694,9 @@ eval_windowaggregates(WindowAggState *winstate)
 	WindowObject agg_winobj;
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot;
+	bool		reduced_frame_set;
+	bool		check_reduced_frame;
+	int			num_rows_in_reduced_frame;
 
 	numaggs = winstate->numaggs;
 	if (numaggs == 0)
@@ -790,6 +814,7 @@ eval_windowaggregates(WindowAggState *winstate)
 			(winstate->frameOptions & FRAMEOPTION_EXCLUSION) ||
 			winstate->aggregatedupto <= winstate->frameheadpos)
 		{
+			elog(DEBUG1, "peraggstate->restart  is set");
 			peraggstate->restart = true;
 			numaggs_restart++;
 		}
@@ -861,8 +886,10 @@ eval_windowaggregates(WindowAggState *winstate)
 	 * If we created a mark pointer for aggregates, keep it pushed up to frame
 	 * head, so that tuplestore can discard unnecessary rows.
 	 */
+#ifdef NOT_USED
 	if (agg_winobj->markptr >= 0)
 		WinSetMarkPosition(agg_winobj, winstate->frameheadpos);
+#endif
 
 	/*
 	 * Now restart the aggregates that require it.
@@ -919,6 +946,10 @@ eval_windowaggregates(WindowAggState *winstate)
 		ExecClearTuple(agg_row_slot);
 	}
 
+	reduced_frame_set = false;
+	check_reduced_frame = false;
+	num_rows_in_reduced_frame = 0;
+
 	/*
 	 * Advance until we reach a row not in frame (or end of partition).
 	 *
@@ -930,12 +961,18 @@ eval_windowaggregates(WindowAggState *winstate)
 	{
 		int			ret;
 
+		elog(DEBUG1, "===== loop in frame starts: " INT64_FORMAT, winstate->aggregatedupto);
+
 		/* Fetch next row if we didn't already */
 		if (TupIsNull(agg_row_slot))
 		{
 			if (!window_gettupleslot(agg_winobj, winstate->aggregatedupto,
 									 agg_row_slot))
+			{
+				if (check_reduced_frame)
+					winstate->aggregatedupto--;
 				break;			/* must be end of partition */
+			}
 		}
 
 		/*
@@ -944,10 +981,47 @@ eval_windowaggregates(WindowAggState *winstate)
 		 */
 		ret = row_is_in_frame(winstate, winstate->aggregatedupto, agg_row_slot);
 		if (ret < 0)
+		{
+			if (winstate->patternVariableList != NIL && check_reduced_frame)
+				winstate->aggregatedupto--;
 			break;
+		}
 		if (ret == 0)
 			goto next_tuple;
 
+		if (winstate->patternVariableList != NIL)
+		{
+			if (!reduced_frame_set)
+			{
+				num_rows_in_reduced_frame = row_is_in_reduced_frame(winstate->agg_winobj, winstate->aggregatedupto);
+				reduced_frame_set = true;
+				elog(DEBUG1, "set num_rows_in_reduced_frame: %d pos: " INT64_FORMAT,
+					 num_rows_in_reduced_frame, winstate->aggregatedupto);
+
+				if (num_rows_in_reduced_frame <= 0)
+					break;
+
+				else if (num_rows_in_reduced_frame > 0)
+					check_reduced_frame = true;
+			}
+
+			if (check_reduced_frame)
+			{
+				elog(DEBUG1, "decrease num_rows_in_reduced_frame: %d pos: " INT64_FORMAT,
+					 num_rows_in_reduced_frame, winstate->aggregatedupto);
+				num_rows_in_reduced_frame--;
+				if (num_rows_in_reduced_frame < 0)
+				{
+					/*
+					 * No more rows remain in the reduced frame. Finish
+					 * accumulating row into the aggregates.
+					 */
+					winstate->aggregatedupto--;
+					break;
+				}
+			}
+		}
+
 		/* Set tuple context for evaluation of aggregate arguments */
 		winstate->tmpcontext->ecxt_outertuple = agg_row_slot;
 
@@ -976,6 +1050,8 @@ next_tuple:
 		ExecClearTuple(agg_row_slot);
 	}
 
+	elog(DEBUG1, "===== break loop in frame starts: " INT64_FORMAT, winstate->aggregatedupto);
+
 	/* The frame's end is not supposed to move backwards, ever */
 	Assert(aggregatedupto_nonrestarted <= winstate->aggregatedupto);
 
@@ -2053,6 +2129,8 @@ ExecWindowAgg(PlanState *pstate)
 
 	CHECK_FOR_INTERRUPTS();
 
+	elog(DEBUG1, "ExecWindowAgg called. pos: " INT64_FORMAT , winstate->currentpos);
+
 	if (winstate->status == WINDOWAGG_DONE)
 		return NULL;
 
@@ -2388,6 +2466,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2567,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2761,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (nargs != 1)
+				continue;
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				continue;
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2831,77 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			elog(DEBUG1, "walker: varattno: %d varattnosyn: %d",attno, attnosyn);
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2919,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2970,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3312,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3100,7 +3332,7 @@ window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 		return false;
 
 	if (pos < winobj->markpos)
-		elog(ERROR, "cannot fetch row before WindowObject's mark position");
+		elog(ERROR, "cannot fetch row: " INT64_FORMAT " before WindowObject's mark position: " INT64_FORMAT,  pos, winobj->markpos );
 
 	oldcontext = MemoryContextSwitchTo(winstate->ss.ps.ps_ExprContext->ecxt_per_query_memory);
 
@@ -3420,14 +3652,54 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+	int			num_reduced_frame;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3494,6 +3766,12 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 						 winstate->frameOptions);
 					break;
 			}
+			num_reduced_frame = row_is_in_reduced_frame(winobj, winstate->frameheadpos);
+			if (num_reduced_frame < 0)
+				goto out_of_frame;
+			else if (num_reduced_frame > 0)
+				if (relpos >= num_reduced_frame)
+					goto out_of_frame;
 			break;
 		case WINDOW_SEEK_TAIL:
 			/* rejecting relpos > 0 is easy and simplifies code below */
@@ -3565,6 +3843,12 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 					mark_pos = 0;	/* keep compiler quiet */
 					break;
 			}
+
+			num_reduced_frame = row_is_in_reduced_frame(winobj, winstate->frameheadpos + relpos);
+			if (num_reduced_frame < 0)
+				goto out_of_frame;
+			else if (num_reduced_frame > 0)
+				abs_pos = winstate->frameheadpos + relpos + num_reduced_frame - 1;
 			break;
 		default:
 			elog(ERROR, "unrecognized window seek type: %d", seektype);
@@ -3583,15 +3867,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3904,400 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
+
+/*
+ * row_is_in_reduced_frame
+ * Determine whether a row is in the current row's reduced window frame according
+ * to row pattern matching
+ *
+ * The row must has been already determined that it is in a full window frame
+ * and fetched it into slot.
+ *
+ * Returns:
+ * = 0, RPR is not defined.
+ * >0, if the row is the first in the reduced frame. Return the number of rows in the reduced frame.
+ * -1, if the row is unmatched row
+ * -2, if the row is in the reduced frame but needed to be skipped because of
+ * AFTER MATCH SKIP PAST LAST ROW
+ */
+static
+int row_is_in_reduced_frame(WindowObject winobj, int64 pos)
+{
+	WindowAggState *winstate = winobj->winstate;
+	ListCell	*lc1, *lc2;
+	bool		expression_result;
+	int			num_matched_rows;
+	int64		original_pos;
+	bool		anymatch;
+	StringInfo	encoded_str;
+	StringInfo	pattern_str = makeStringInfo();
+
+	/*
+	 * Array of pattern variables evaluted to true.
+	 * Each character corresponds to pattern variable.
+	 * Example:
+	 * str_set[0] = "AB";
+	 * str_set[1] = "AC";
+	 * In this case at row 0 A and B are true, and A and C are true in row 1.
+	 */
+	#define ENCODED_STR_ARRAY_ALLOC_SIZE	128
+	StringInfo	*str_set = NULL;
+	int		str_set_index;
+	int		str_set_size;
+
+	if (winstate->patternVariableList == NIL)
+	{
+		/*
+		 * RPR is not defined. Assume that we are always in the the reduced
+		 * window frame.
+		 */
+		return 0;
+	}
+
+	/* save original pos */
+	original_pos = pos;
+
+	/*
+	 * Check whether the row speicied by pos is in the reduced frame. The
+	 * second and subsequent rows need to be recognized as "unmatched" rows if
+	 * AFTER MATCH SKIP PAST LAST ROW is defined.
+	 */
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW &&
+		pos > winstate->headpos_in_reduced_frame &&
+		pos < (winstate->headpos_in_reduced_frame + winstate->num_rows_in_reduced_frame))
+		return -2;
+		
+	/*
+	 * Loop over until none of pattern matches or encounters end of frame.
+	 */
+	for (;;)
+	{
+		int64	result_pos = -1;
+
+		/*
+		 * Loop over each PATTERN variable.
+		 */
+		anymatch = false;
+		encoded_str = makeStringInfo();
+
+		forboth(lc1, winstate->patternVariableList, lc2, winstate->patternRegexpList)
+		{
+			char	*vname = strVal(lfirst(lc1));
+			char	*quantifier = strVal(lfirst(lc2));
+
+			elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s quantifier: %s", pos, vname, quantifier);
+
+			expression_result = false;
+
+			/* evaluate row pattern against current row */
+			result_pos = evaluate_pattern(winobj, pos, vname, encoded_str, &expression_result);
+			if (expression_result)
+			{
+				elog(DEBUG1, "expression result is true");
+				anymatch = true;
+			}
+
+			/*
+			 * If out of frame, we are done.
+			 */
+			 if (result_pos < 0)
+				 break;
+		}
+
+		if (!anymatch)
+		{
+			/* none of patterns matched. */
+			break;
+		}
+
+		/* build encoded string array */
+		if (str_set == NULL)
+		{
+			str_set_index = 0;
+			str_set_size = ENCODED_STR_ARRAY_ALLOC_SIZE * sizeof(StringInfo);
+			str_set = palloc(str_set_size);
+		}
+
+		str_set[str_set_index++] = encoded_str;
+
+		elog(DEBUG1, "pos: " INT64_FORMAT " str_set_index: %d encoded_str: %s", pos, str_set_index, encoded_str->data);
+
+		if (str_set_index >= str_set_size)
+		{
+			str_set_size *= 2;
+			str_set = repalloc(str_set, str_set_size);
+		}
+
+		/* move to next row */
+		pos++;
+
+		if (result_pos < 0)
+		{
+			/* out of frame */
+			break;
+		}
+	}
+
+	if (str_set == NULL)
+	{
+		/* no matches found in the first row */
+		return -1;
+	}
+
+	elog(DEBUG1, "pos: " INT64_FORMAT " encoded_str: %s", pos, encoded_str->data);
+
+	/* build regular expression */
+	pattern_str = makeStringInfo();
+	appendStringInfoChar(pattern_str, '^');
+	forboth(lc1, winstate->patternVariableList, lc2, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc1));
+		char	*quantifier = strVal(lfirst(lc2));
+
+		appendStringInfoChar(pattern_str, vname[0]);
+		if (quantifier[0])
+			appendStringInfoChar(pattern_str, quantifier[0]);
+		elog(DEBUG1, "vname: %s quantifier: %s", vname, quantifier);
+	}
+
+	elog(DEBUG1, "pos: " INT64_FORMAT " pattern: %s", pos, pattern_str->data);
+
+	/* look for matching pattern variable sequence */
+	num_matched_rows = search_str_set(pattern_str->data, str_set, str_set_index);
+	if (num_matched_rows <= 0)
+		return -1;
+
+	/*
+	 * We are at the first row in the reduced frame.  Save the number of
+	 * matched rows as the number of rows in the reduced frame.
+	 */
+	winstate->headpos_in_reduced_frame = original_pos;
+	winstate->num_rows_in_reduced_frame = num_matched_rows;
+
+	return num_matched_rows;
+}
+
+/*
+ * search set of encode_str.
+ * set_size: size of set_str array.
+ */
+static
+int search_str_set(char *pattern, StringInfo *str_set, int set_size)
+{
+	char		*encoded_str = palloc0(set_size+1);
+	int			resultlen = 0;
+
+	search_str_set_recurse(pattern, str_set, set_size, 0, encoded_str, &resultlen);
+	elog(DEBUG1, "search_str_set returns %d", resultlen);
+	return resultlen;
+}
+
+static
+void search_str_set_recurse(char *pattern, StringInfo *str_set, int set_size, int set_index, char *encoded_str, int *resultlen)
+{
+	char	*p;
+
+	if (set_index >= set_size)
+	{
+		Datum	d;
+		text	*res;
+		char	*substr;
+
+		/*
+		 * We first perform pattern matching using regexp_instr, then call
+		 * textregexsubstr to get matched substring to know how log the
+		 * matched string is. That is the number of rows in the reduced window
+		 * frame.  The reason why we can't call textregexsubstr is, it error
+		 * out if pattern is not match.
+		 */
+		if (DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+												  PointerGetDatum(cstring_to_text(encoded_str)),
+												  PointerGetDatum(cstring_to_text(pattern)))) > 0)
+		{
+			d = DirectFunctionCall2Coll(textregexsubstr,
+										DEFAULT_COLLATION_OID,
+										PointerGetDatum(cstring_to_text(encoded_str)),
+										PointerGetDatum(cstring_to_text(pattern)));
+			if (d != 0)
+			{
+				int		len;
+
+				res = DatumGetTextPP(d);
+				substr = text_to_cstring(res);
+				len = strlen(substr);
+				if (len > *resultlen)
+					/* remember the longest match */
+					*resultlen = len;
+			}
+		}
+		return;
+	}
+
+	p = str_set[set_index]->data;
+	while (*p)
+	{
+		encoded_str[set_index] = *p;
+		p++;
+		search_str_set_recurse(pattern, str_set, set_size, set_index + 1, encoded_str, resultlen);
+	}
+}
+
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match absolute row position
+ * other wise out of frame.
+ */
+static
+int64 evaluate_pattern(WindowObject winobj, int64 current_pos, 
+						char *vname, StringInfo encoded_str, bool *result)
+{
+	WindowAggState	*winstate = winobj->winstate;
+	ExprContext		*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell		*lc1, *lc2;
+	ExprState		*pat;
+	Datum			eval_result;
+	bool			out_of_frame = false;
+	bool			isnull;
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		/* get current, previous and next tuples */
+		if (!get_slots(winobj, current_pos))
+		{
+			out_of_frame = true;
+		}
+		else
+		{
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: " INT64_FORMAT, vname, current_pos);
+				*result = false;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: " INT64_FORMAT, vname, current_pos);
+					*result = false;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: " INT64_FORMAT, vname, current_pos);
+					appendStringInfoChar(encoded_str, vname[0]);
+					*result = true;
+				}
+			}
+			break;
+		}
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+	}
+	return current_pos;
+}
+
+/*
+ * Get current, previous and next tuples.
+ * Returns false if current row is out of partition/full frame.
+ */
+static
+bool get_slots(WindowObject winobj, int64 current_pos)
+{
+	WindowAggState *winstate = winobj->winstate;
+	TupleTableSlot *slot;
+	int		ret;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* set up current row tuple slot */
+	slot = winstate->temp_slot_1;
+	if (!window_gettupleslot(winobj, current_pos, slot))
+	{
+		elog(DEBUG1, "current row is out of partition at:" INT64_FORMAT, current_pos);
+		return false;
+
+		ret = row_is_in_frame(winstate, current_pos, slot);
+		if (ret <= 0)
+		{
+			elog(DEBUG1, "current row is out of frame at: " INT64_FORMAT, current_pos);
+			return false;
+		}
+	}
+	econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		if (!window_gettupleslot(winobj, current_pos - 1, slot))
+		{
+			elog(DEBUG1, "previous row is out of partition at: " INT64_FORMAT, current_pos - 1);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		else
+		{
+			ret = row_is_in_frame(winstate, current_pos - 1, slot);
+			if (ret <= 0)
+			{
+				elog(DEBUG1, "previous row is out of frame at: " INT64_FORMAT, current_pos - 1);
+				econtext->ecxt_outertuple = winstate->null_slot;
+			}
+			else
+			{
+				econtext->ecxt_outertuple = slot;
+			}
+		}
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	if (!window_gettupleslot(winobj, current_pos + 1, slot))
+	{
+		elog(DEBUG1, "next row is out of partiton at: " INT64_FORMAT, current_pos + 1);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+	{
+		ret = row_is_in_frame(winstate, current_pos + 1, slot);
+		if (ret <= 0)
+		{
+			elog(DEBUG1, "next row is out of frame at: " INT64_FORMAT, current_pos + 1);
+			econtext->ecxt_innertuple = winstate->null_slot;
+		}
+		else
+			econtext->ecxt_innertuple = slot;
+	}
+	return true;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..e4cab36ec9 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,11 +39,19 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
 
-
 /*
  * utility routine for *_rank functions.
  */
@@ -673,7 +684,7 @@ window_last_value(PG_FUNCTION_ARGS)
 	bool		isnull;
 
 	result = WinGetFuncArgInFrame(winobj, 0,
-								  0, WINDOW_SEEK_TAIL, true,
+								  0, WINDOW_SEEK_TAIL, false,
 								  &isnull, NULL);
 	if (isnull)
 		PG_RETURN_NULL();
@@ -713,3 +724,26 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..fa100b2665 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,12 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6123', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..4fd3bd1a93 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,16 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
+
+	/* head of the reduced window frame */
+	int64		headpos_in_reduced_frame;
+	/* number of rows in the reduced window frame */
+	int64		num_rows_in_reduced_frame;
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..1e292648e9 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,15 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Wed_Jul_26_21_21_34_2023_317)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v3-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread

* [PATCH v3 4/7] Row pattern recognition patch (executor).
@ 2023-07-26 10:49 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 108+ messages in thread

From: Tatsuo Ishii @ 2023-07-26 10:49 UTC (permalink / raw)

---
 src/backend/executor/nodeWindowAgg.c | 701 ++++++++++++++++++++++++++-
 src/backend/utils/adt/windowfuncs.c  |  38 +-
 src/include/catalog/pg_proc.dat      |   6 +
 src/include/nodes/execnodes.h        |  18 +
 src/include/windowapi.h              |   8 +
 5 files changed, 758 insertions(+), 13 deletions(-)

diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 310ac23e3a..0586bf57d6 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -36,6 +36,7 @@
 #include "access/htup_details.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_aggregate.h"
+#include "catalog/pg_collation_d.h"
 #include "catalog/pg_proc.h"
 #include "executor/executor.h"
 #include "executor/nodeWindowAgg.h"
@@ -48,6 +49,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
+#include "utils/fmgroids.h"
 #include "utils/expandeddatum.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -159,6 +161,14 @@ typedef struct WindowStatePerAggData
 	bool		restart;		/* need to restart this agg in this cycle? */
 } WindowStatePerAggData;
 
+/*
+ * Map between Var attno in a target list and the parsed attno.
+ */
+typedef struct AttnoMap {
+	List		*attno;			/* att number in target list (list of AttNumber) */
+	List		*attnosyn;		/* parsed att number (list of AttNumber) */
+} AttnoMap;
+
 static void initialize_windowaggregate(WindowAggState *winstate,
 									   WindowStatePerFunc perfuncstate,
 									   WindowStatePerAgg peraggstate);
@@ -182,8 +192,9 @@ static void begin_partition(WindowAggState *winstate);
 static void spool_tuples(WindowAggState *winstate, int64 pos);
 static void release_partition(WindowAggState *winstate);
 
-static int	row_is_in_frame(WindowAggState *winstate, int64 pos,
+static int  row_is_in_frame(WindowAggState *winstate, int64 pos,
 							TupleTableSlot *slot);
+
 static void update_frameheadpos(WindowAggState *winstate);
 static void update_frametailpos(WindowAggState *winstate);
 static void update_grouptailpos(WindowAggState *winstate);
@@ -195,9 +206,19 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
 
 static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
 					  TupleTableSlot *slot2);
-static bool window_gettupleslot(WindowObject winobj, int64 pos,
-								TupleTableSlot *slot);
 
+static void attno_map(Node *node, AttnoMap *map);
+static bool attno_map_walker(Node *node, void *context);
+static int row_is_in_reduced_frame(WindowObject winobj, int64 pos);
+
+static int64 evaluate_pattern(WindowObject winobj, int64 current_pos, 
+							  char *vname, StringInfo encoded_str, bool *result);
+
+static bool get_slots(WindowObject winobj, int64 current_pos);
+
+static int search_str_set(char *pattern, StringInfo *str_set, int set_size);
+static void search_str_set_recurse(char *pattern, StringInfo *str_set, int set_size, int set_index,
+								   char *encoded_str, int *resultlen);
 
 /*
  * initialize_windowaggregate
@@ -673,6 +694,9 @@ eval_windowaggregates(WindowAggState *winstate)
 	WindowObject agg_winobj;
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot;
+	bool		reduced_frame_set;
+	bool		check_reduced_frame;
+	int			num_rows_in_reduced_frame;
 
 	numaggs = winstate->numaggs;
 	if (numaggs == 0)
@@ -790,6 +814,7 @@ eval_windowaggregates(WindowAggState *winstate)
 			(winstate->frameOptions & FRAMEOPTION_EXCLUSION) ||
 			winstate->aggregatedupto <= winstate->frameheadpos)
 		{
+			elog(DEBUG1, "peraggstate->restart  is set");
 			peraggstate->restart = true;
 			numaggs_restart++;
 		}
@@ -861,8 +886,10 @@ eval_windowaggregates(WindowAggState *winstate)
 	 * If we created a mark pointer for aggregates, keep it pushed up to frame
 	 * head, so that tuplestore can discard unnecessary rows.
 	 */
+#ifdef NOT_USED
 	if (agg_winobj->markptr >= 0)
 		WinSetMarkPosition(agg_winobj, winstate->frameheadpos);
+#endif
 
 	/*
 	 * Now restart the aggregates that require it.
@@ -919,6 +946,10 @@ eval_windowaggregates(WindowAggState *winstate)
 		ExecClearTuple(agg_row_slot);
 	}
 
+	reduced_frame_set = false;
+	check_reduced_frame = false;
+	num_rows_in_reduced_frame = 0;
+
 	/*
 	 * Advance until we reach a row not in frame (or end of partition).
 	 *
@@ -930,12 +961,18 @@ eval_windowaggregates(WindowAggState *winstate)
 	{
 		int			ret;
 
+		elog(DEBUG1, "===== loop in frame starts: " INT64_FORMAT, winstate->aggregatedupto);
+
 		/* Fetch next row if we didn't already */
 		if (TupIsNull(agg_row_slot))
 		{
 			if (!window_gettupleslot(agg_winobj, winstate->aggregatedupto,
 									 agg_row_slot))
+			{
+				if (check_reduced_frame)
+					winstate->aggregatedupto--;
 				break;			/* must be end of partition */
+			}
 		}
 
 		/*
@@ -944,10 +981,47 @@ eval_windowaggregates(WindowAggState *winstate)
 		 */
 		ret = row_is_in_frame(winstate, winstate->aggregatedupto, agg_row_slot);
 		if (ret < 0)
+		{
+			if (winstate->patternVariableList != NIL && check_reduced_frame)
+				winstate->aggregatedupto--;
 			break;
+		}
 		if (ret == 0)
 			goto next_tuple;
 
+		if (winstate->patternVariableList != NIL)
+		{
+			if (!reduced_frame_set)
+			{
+				num_rows_in_reduced_frame = row_is_in_reduced_frame(winstate->agg_winobj, winstate->aggregatedupto);
+				reduced_frame_set = true;
+				elog(DEBUG1, "set num_rows_in_reduced_frame: %d pos: " INT64_FORMAT,
+					 num_rows_in_reduced_frame, winstate->aggregatedupto);
+
+				if (num_rows_in_reduced_frame <= 0)
+					break;
+
+				else if (num_rows_in_reduced_frame > 0)
+					check_reduced_frame = true;
+			}
+
+			if (check_reduced_frame)
+			{
+				elog(DEBUG1, "decrease num_rows_in_reduced_frame: %d pos: " INT64_FORMAT,
+					 num_rows_in_reduced_frame, winstate->aggregatedupto);
+				num_rows_in_reduced_frame--;
+				if (num_rows_in_reduced_frame < 0)
+				{
+					/*
+					 * No more rows remain in the reduced frame. Finish
+					 * accumulating row into the aggregates.
+					 */
+					winstate->aggregatedupto--;
+					break;
+				}
+			}
+		}
+
 		/* Set tuple context for evaluation of aggregate arguments */
 		winstate->tmpcontext->ecxt_outertuple = agg_row_slot;
 
@@ -976,6 +1050,8 @@ next_tuple:
 		ExecClearTuple(agg_row_slot);
 	}
 
+	elog(DEBUG1, "===== break loop in frame starts: " INT64_FORMAT, winstate->aggregatedupto);
+
 	/* The frame's end is not supposed to move backwards, ever */
 	Assert(aggregatedupto_nonrestarted <= winstate->aggregatedupto);
 
@@ -2053,6 +2129,8 @@ ExecWindowAgg(PlanState *pstate)
 
 	CHECK_FOR_INTERRUPTS();
 
+	elog(DEBUG1, "ExecWindowAgg called. pos: " INT64_FORMAT , winstate->currentpos);
+
 	if (winstate->status == WINDOWAGG_DONE)
 		return NULL;
 
@@ -2388,6 +2466,12 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	TupleDesc	scanDesc;
 	ListCell   *l;
 
+	TargetEntry	*te;
+	Expr		*expr;
+	Var			*var;
+	int			nargs;
+	AttnoMap	attnomap;
+
 	/* check for unsupported flags */
 	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
 
@@ -2483,6 +2567,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc,
 												   &TTSOpsMinimalTuple);
 
+	winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+
+	winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc,
+												 &TTSOpsMinimalTuple);
+	winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot);
+
 	/*
 	 * create frame head and tail slots only if needed (must create slots in
 	 * exactly the same cases that update_frameheadpos and update_frametailpos
@@ -2667,6 +2761,69 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	winstate->inRangeAsc = node->inRangeAsc;
 	winstate->inRangeNullsFirst = node->inRangeNullsFirst;
 
+	/* Set up SKIP TO type */
+	winstate->rpSkipTo = node->rpSkipTo;
+	/* Set up row pattern recognition PATTERN clause */
+	winstate->patternVariableList = node->patternVariable;
+	winstate->patternRegexpList = node->patternRegexp;
+
+	/* Set up row pattern recognition DEFINE clause */
+
+	/*
+	 * Collect mapping between varattno and varattnosyn in the targetlist.
+	 * XXX: For now we only check RPR's argument. Eventually we have to
+	 * recurse the targetlist to find out all mappings in Var nodes.
+	 */
+	attnomap.attno = NIL;
+	attnomap.attnosyn = NIL;
+
+	foreach (l, node->plan.targetlist)
+	{
+		te = lfirst(l);
+		if (IsA(te->expr, WindowFunc))
+		{
+			WindowFunc	*func = (WindowFunc *)te->expr;
+
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (nargs != 1)
+				continue;
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				continue;
+
+			var = (Var *)expr;
+			elog(DEBUG1, "resname: %s varattno: %d varattnosyn: %d",
+				 te->resname, var->varattno, var->varattnosyn);
+			attnomap.attno = lappend_int(attnomap.attno, var->varattno);
+			attnomap.attnosyn = lappend_int(attnomap.attnosyn, var->varattnosyn);
+		}
+	}
+
+	winstate->defineVariableList = NIL;
+	winstate->defineClauseList = NIL;
+	if (node->defineClause != NIL)
+	{
+		foreach(l, node->defineClause)
+		{
+			char		*name;
+			ExprState	*exps;
+
+			te = lfirst(l);
+			name = te->resname;
+			expr = te->expr;
+
+			elog(DEBUG1, "defineVariable name: %s", name);
+			winstate->defineVariableList = lappend(winstate->defineVariableList,
+												   makeString(pstrdup(name)));
+			/* tweak expr so that it referes to outer slot */
+			attno_map((Node *)expr, &attnomap);
+			exps = ExecInitExpr(expr, (PlanState *) winstate);
+			winstate->defineClauseList = lappend(winstate->defineClauseList, exps);
+		}
+	}
+
 	winstate->all_first = true;
 	winstate->partition_spooled = false;
 	winstate->more_partitions = false;
@@ -2674,6 +2831,77 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
 	return winstate;
 }
 
+/*
+ * Rewrite Var node's varattno to the varattno which is used in the target
+ * list using AttnoMap.  We also rewrite varno so that it sees outer tuple
+ * (PREV) or inner tuple (NEXT).
+ */
+static void
+attno_map(Node *node, AttnoMap *map)
+{
+	(void) expression_tree_walker(node, attno_map_walker, (void *) map);
+}
+
+static bool
+attno_map_walker(Node *node, void *context)
+{
+	FuncExpr	*func;
+	int			nargs;
+	Expr		*expr;
+	Var			*var;
+	AttnoMap	*attnomap;
+	ListCell	*lc1, *lc2;
+	
+	if (node == NULL)
+		return false;
+
+	attnomap = (AttnoMap *) context;
+
+	if (IsA(node, FuncExpr))
+	{
+		func = (FuncExpr *)node;
+
+		if (func->funcid == F_PREV || func->funcid == F_NEXT)
+		{
+			/* sanity check */
+			nargs = list_length(func->args);
+			if (list_length(func->args) != 1)
+				elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", func->funcid, nargs);
+
+			expr = (Expr *) lfirst(list_head(func->args));
+			if (!IsA(expr, Var))
+				elog(ERROR, "PREV/NEXT's arg is not Var");	/* XXX: is it possible that arg type is Const? */
+			var = (Var *)expr;
+
+			if (func->funcid == F_PREV)
+				var->varno = OUTER_VAR;
+			else
+				var->varno = INNER_VAR;
+		}
+		return expression_tree_walker(node, attno_map_walker, (void *) context);
+	}
+	else if (IsA(node, Var))
+	{
+		var = (Var *)node;	 
+
+		elog(DEBUG1, "original varno: %d varattno: %d", var->varno, var->varattno);
+
+		forboth(lc1, attnomap->attno, lc2, attnomap->attnosyn)
+		{
+			int	attno = lfirst_int(lc1);
+			int	attnosyn = lfirst_int(lc2);
+
+			elog(DEBUG1, "walker: varattno: %d varattnosyn: %d",attno, attnosyn);
+			if (var->varattno == attnosyn)
+			{
+				elog(DEBUG1, "loc: %d rewrite varattno from: %d to %d", var->location, attnosyn, attno);
+				var->varattno = attno;
+			}
+		}
+	}
+	return expression_tree_walker(node, attno_map_walker, (void *) context);
+}
+
 /* -----------------
  * ExecEndWindowAgg
  * -----------------
@@ -2691,6 +2919,8 @@ ExecEndWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -2740,6 +2970,8 @@ ExecReScanWindowAgg(WindowAggState *node)
 	ExecClearTuple(node->agg_row_slot);
 	ExecClearTuple(node->temp_slot_1);
 	ExecClearTuple(node->temp_slot_2);
+	ExecClearTuple(node->prev_slot);
+	ExecClearTuple(node->next_slot);
 	if (node->framehead_slot)
 		ExecClearTuple(node->framehead_slot);
 	if (node->frametail_slot)
@@ -3080,7 +3312,7 @@ are_peers(WindowAggState *winstate, TupleTableSlot *slot1,
  *
  * Returns true if successful, false if no such row
  */
-static bool
+bool
 window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 {
 	WindowAggState *winstate = winobj->winstate;
@@ -3100,7 +3332,7 @@ window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot)
 		return false;
 
 	if (pos < winobj->markpos)
-		elog(ERROR, "cannot fetch row before WindowObject's mark position");
+		elog(ERROR, "cannot fetch row: " INT64_FORMAT " before WindowObject's mark position: " INT64_FORMAT,  pos, winobj->markpos );
 
 	oldcontext = MemoryContextSwitchTo(winstate->ss.ps.ps_ExprContext->ecxt_per_query_memory);
 
@@ -3420,14 +3652,54 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 	WindowAggState *winstate;
 	ExprContext *econtext;
 	TupleTableSlot *slot;
-	int64		abs_pos;
-	int64		mark_pos;
 
 	Assert(WindowObjectIsValid(winobj));
 	winstate = winobj->winstate;
 	econtext = winstate->ss.ps.ps_ExprContext;
 	slot = winstate->temp_slot_1;
 
+	if (WinGetSlotInFrame(winobj, slot,
+						  relpos, seektype, set_mark,
+						  isnull, isout) == 0)
+	{
+		econtext->ecxt_outertuple = slot;
+		return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
+							econtext, isnull);
+	}
+
+	if (isout)
+		*isout = true;
+	*isnull = true;
+	return (Datum) 0;
+}
+
+/*
+ * WinGetSlotInFrame
+ * slot: TupleTableSlot to store the result
+ * relpos: signed rowcount offset from the seek position
+ * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL
+ * set_mark: If the row is found/in frame and set_mark is true, the mark is
+ *		moved to the row as a side-effect.
+ * isnull: output argument, receives isnull status of result
+ * isout: output argument, set to indicate whether target row position
+ *		is out of frame (can pass NULL if caller doesn't care about this)
+ *
+ * Returns 0 if we successfullt got the slot. false if out of frame.
+ * (also isout is set)
+ */
+int
+WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+					 int relpos, int seektype, bool set_mark,
+					 bool *isnull, bool *isout)
+{
+	WindowAggState *winstate;
+	int64		abs_pos;
+	int64		mark_pos;
+	int			num_reduced_frame;
+
+	Assert(WindowObjectIsValid(winobj));
+	winstate = winobj->winstate;
+
 	switch (seektype)
 	{
 		case WINDOW_SEEK_CURRENT:
@@ -3494,6 +3766,12 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 						 winstate->frameOptions);
 					break;
 			}
+			num_reduced_frame = row_is_in_reduced_frame(winobj, winstate->frameheadpos);
+			if (num_reduced_frame < 0)
+				goto out_of_frame;
+			else if (num_reduced_frame > 0)
+				if (relpos >= num_reduced_frame)
+					goto out_of_frame;
 			break;
 		case WINDOW_SEEK_TAIL:
 			/* rejecting relpos > 0 is easy and simplifies code below */
@@ -3565,6 +3843,12 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 					mark_pos = 0;	/* keep compiler quiet */
 					break;
 			}
+
+			num_reduced_frame = row_is_in_reduced_frame(winobj, winstate->frameheadpos + relpos);
+			if (num_reduced_frame < 0)
+				goto out_of_frame;
+			else if (num_reduced_frame > 0)
+				abs_pos = winstate->frameheadpos + relpos + num_reduced_frame - 1;
 			break;
 		default:
 			elog(ERROR, "unrecognized window seek type: %d", seektype);
@@ -3583,15 +3867,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno,
 		*isout = false;
 	if (set_mark)
 		WinSetMarkPosition(winobj, mark_pos);
-	econtext->ecxt_outertuple = slot;
-	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
-						econtext, isnull);
+	return 0;
 
 out_of_frame:
 	if (isout)
 		*isout = true;
 	*isnull = true;
-	return (Datum) 0;
+	return -1;
 }
 
 /*
@@ -3622,3 +3904,400 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull)
 	return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno),
 						econtext, isnull);
 }
+
+WindowAggState *
+WinGetAggState(WindowObject winobj)
+{
+	return winobj->winstate;
+}
+
+/*
+ * row_is_in_reduced_frame
+ * Determine whether a row is in the current row's reduced window frame according
+ * to row pattern matching
+ *
+ * The row must has been already determined that it is in a full window frame
+ * and fetched it into slot.
+ *
+ * Returns:
+ * = 0, RPR is not defined.
+ * >0, if the row is the first in the reduced frame. Return the number of rows in the reduced frame.
+ * -1, if the row is unmatched row
+ * -2, if the row is in the reduced frame but needed to be skipped because of
+ * AFTER MATCH SKIP PAST LAST ROW
+ */
+static
+int row_is_in_reduced_frame(WindowObject winobj, int64 pos)
+{
+	WindowAggState *winstate = winobj->winstate;
+	ListCell	*lc1, *lc2;
+	bool		expression_result;
+	int			num_matched_rows;
+	int64		original_pos;
+	bool		anymatch;
+	StringInfo	encoded_str;
+	StringInfo	pattern_str = makeStringInfo();
+
+	/*
+	 * Array of pattern variables evaluted to true.
+	 * Each character corresponds to pattern variable.
+	 * Example:
+	 * str_set[0] = "AB";
+	 * str_set[1] = "AC";
+	 * In this case at row 0 A and B are true, and A and C are true in row 1.
+	 */
+	#define ENCODED_STR_ARRAY_ALLOC_SIZE	128
+	StringInfo	*str_set = NULL;
+	int		str_set_index;
+	int		str_set_size;
+
+	if (winstate->patternVariableList == NIL)
+	{
+		/*
+		 * RPR is not defined. Assume that we are always in the the reduced
+		 * window frame.
+		 */
+		return 0;
+	}
+
+	/* save original pos */
+	original_pos = pos;
+
+	/*
+	 * Check whether the row speicied by pos is in the reduced frame. The
+	 * second and subsequent rows need to be recognized as "unmatched" rows if
+	 * AFTER MATCH SKIP PAST LAST ROW is defined.
+	 */
+	if (winstate->rpSkipTo == ST_PAST_LAST_ROW &&
+		pos > winstate->headpos_in_reduced_frame &&
+		pos < (winstate->headpos_in_reduced_frame + winstate->num_rows_in_reduced_frame))
+		return -2;
+		
+	/*
+	 * Loop over until none of pattern matches or encounters end of frame.
+	 */
+	for (;;)
+	{
+		int64	result_pos = -1;
+
+		/*
+		 * Loop over each PATTERN variable.
+		 */
+		anymatch = false;
+		encoded_str = makeStringInfo();
+
+		forboth(lc1, winstate->patternVariableList, lc2, winstate->patternRegexpList)
+		{
+			char	*vname = strVal(lfirst(lc1));
+			char	*quantifier = strVal(lfirst(lc2));
+
+			elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s quantifier: %s", pos, vname, quantifier);
+
+			expression_result = false;
+
+			/* evaluate row pattern against current row */
+			result_pos = evaluate_pattern(winobj, pos, vname, encoded_str, &expression_result);
+			if (expression_result)
+			{
+				elog(DEBUG1, "expression result is true");
+				anymatch = true;
+			}
+
+			/*
+			 * If out of frame, we are done.
+			 */
+			 if (result_pos < 0)
+				 break;
+		}
+
+		if (!anymatch)
+		{
+			/* none of patterns matched. */
+			break;
+		}
+
+		/* build encoded string array */
+		if (str_set == NULL)
+		{
+			str_set_index = 0;
+			str_set_size = ENCODED_STR_ARRAY_ALLOC_SIZE * sizeof(StringInfo);
+			str_set = palloc(str_set_size);
+		}
+
+		str_set[str_set_index++] = encoded_str;
+
+		elog(DEBUG1, "pos: " INT64_FORMAT " str_set_index: %d encoded_str: %s", pos, str_set_index, encoded_str->data);
+
+		if (str_set_index >= str_set_size)
+		{
+			str_set_size *= 2;
+			str_set = repalloc(str_set, str_set_size);
+		}
+
+		/* move to next row */
+		pos++;
+
+		if (result_pos < 0)
+		{
+			/* out of frame */
+			break;
+		}
+	}
+
+	if (str_set == NULL)
+	{
+		/* no matches found in the first row */
+		return -1;
+	}
+
+	elog(DEBUG1, "pos: " INT64_FORMAT " encoded_str: %s", pos, encoded_str->data);
+
+	/* build regular expression */
+	pattern_str = makeStringInfo();
+	appendStringInfoChar(pattern_str, '^');
+	forboth(lc1, winstate->patternVariableList, lc2, winstate->patternRegexpList)
+	{
+		char	*vname = strVal(lfirst(lc1));
+		char	*quantifier = strVal(lfirst(lc2));
+
+		appendStringInfoChar(pattern_str, vname[0]);
+		if (quantifier[0])
+			appendStringInfoChar(pattern_str, quantifier[0]);
+		elog(DEBUG1, "vname: %s quantifier: %s", vname, quantifier);
+	}
+
+	elog(DEBUG1, "pos: " INT64_FORMAT " pattern: %s", pos, pattern_str->data);
+
+	/* look for matching pattern variable sequence */
+	num_matched_rows = search_str_set(pattern_str->data, str_set, str_set_index);
+	if (num_matched_rows <= 0)
+		return -1;
+
+	/*
+	 * We are at the first row in the reduced frame.  Save the number of
+	 * matched rows as the number of rows in the reduced frame.
+	 */
+	winstate->headpos_in_reduced_frame = original_pos;
+	winstate->num_rows_in_reduced_frame = num_matched_rows;
+
+	return num_matched_rows;
+}
+
+/*
+ * search set of encode_str.
+ * set_size: size of set_str array.
+ */
+static
+int search_str_set(char *pattern, StringInfo *str_set, int set_size)
+{
+	char		*encoded_str = palloc0(set_size+1);
+	int			resultlen = 0;
+
+	search_str_set_recurse(pattern, str_set, set_size, 0, encoded_str, &resultlen);
+	elog(DEBUG1, "search_str_set returns %d", resultlen);
+	return resultlen;
+}
+
+static
+void search_str_set_recurse(char *pattern, StringInfo *str_set, int set_size, int set_index, char *encoded_str, int *resultlen)
+{
+	char	*p;
+
+	if (set_index >= set_size)
+	{
+		Datum	d;
+		text	*res;
+		char	*substr;
+
+		/*
+		 * We first perform pattern matching using regexp_instr, then call
+		 * textregexsubstr to get matched substring to know how log the
+		 * matched string is. That is the number of rows in the reduced window
+		 * frame.  The reason why we can't call textregexsubstr is, it error
+		 * out if pattern is not match.
+		 */
+		if (DatumGetInt32(DirectFunctionCall2Coll(regexp_instr, DEFAULT_COLLATION_OID,
+												  PointerGetDatum(cstring_to_text(encoded_str)),
+												  PointerGetDatum(cstring_to_text(pattern)))) > 0)
+		{
+			d = DirectFunctionCall2Coll(textregexsubstr,
+										DEFAULT_COLLATION_OID,
+										PointerGetDatum(cstring_to_text(encoded_str)),
+										PointerGetDatum(cstring_to_text(pattern)));
+			if (d != 0)
+			{
+				int		len;
+
+				res = DatumGetTextPP(d);
+				substr = text_to_cstring(res);
+				len = strlen(substr);
+				if (len > *resultlen)
+					/* remember the longest match */
+					*resultlen = len;
+			}
+		}
+		return;
+	}
+
+	p = str_set[set_index]->data;
+	while (*p)
+	{
+		encoded_str[set_index] = *p;
+		p++;
+		search_str_set_recurse(pattern, str_set, set_size, set_index + 1, encoded_str, resultlen);
+	}
+}
+
+
+/*
+ * Evaluate expression associated with PATTERN variable vname.
+ * relpos is relative row position in a frame (starting from 0).
+ * "quantifier" is the quatifier part of the PATTERN regular expression.
+ * Currently only '+' is allowed.
+ * result is out paramater representing the expression evaluation result
+ * is true of false.
+ * Return values are:
+ * >=0: the last match absolute row position
+ * other wise out of frame.
+ */
+static
+int64 evaluate_pattern(WindowObject winobj, int64 current_pos, 
+						char *vname, StringInfo encoded_str, bool *result)
+{
+	WindowAggState	*winstate = winobj->winstate;
+	ExprContext		*econtext = winstate->ss.ps.ps_ExprContext;
+	ListCell		*lc1, *lc2;
+	ExprState		*pat;
+	Datum			eval_result;
+	bool			out_of_frame = false;
+	bool			isnull;
+
+	forboth (lc1, winstate->defineVariableList, lc2, winstate->defineClauseList)
+	{
+		char	*name = strVal(lfirst(lc1));
+
+		if (strcmp(vname, name))
+			continue;
+
+		/* set expression to evaluate */
+		pat = lfirst(lc2);
+
+		/* get current, previous and next tuples */
+		if (!get_slots(winobj, current_pos))
+		{
+			out_of_frame = true;
+		}
+		else
+		{
+			/* evaluate the expression */
+			eval_result = ExecEvalExpr(pat, econtext, &isnull);
+			if (isnull)
+			{
+				/* expression is NULL */
+				elog(DEBUG1, "expression for %s is NULL at row: " INT64_FORMAT, vname, current_pos);
+				*result = false;
+			}
+			else
+			{
+				if (!DatumGetBool(eval_result))
+				{
+					/* expression is false */
+					elog(DEBUG1, "expression for %s is false at row: " INT64_FORMAT, vname, current_pos);
+					*result = false;
+				}
+				else
+				{
+					/* expression is true */
+					elog(DEBUG1, "expression for %s is true at row: " INT64_FORMAT, vname, current_pos);
+					appendStringInfoChar(encoded_str, vname[0]);
+					*result = true;
+				}
+			}
+			break;
+		}
+
+		if (out_of_frame)
+		{
+			*result = false;
+			return -1;
+		}
+	}
+	return current_pos;
+}
+
+/*
+ * Get current, previous and next tuples.
+ * Returns false if current row is out of partition/full frame.
+ */
+static
+bool get_slots(WindowObject winobj, int64 current_pos)
+{
+	WindowAggState *winstate = winobj->winstate;
+	TupleTableSlot *slot;
+	int		ret;
+	ExprContext *econtext;
+
+	econtext = winstate->ss.ps.ps_ExprContext;
+
+	/* set up current row tuple slot */
+	slot = winstate->temp_slot_1;
+	if (!window_gettupleslot(winobj, current_pos, slot))
+	{
+		elog(DEBUG1, "current row is out of partition at:" INT64_FORMAT, current_pos);
+		return false;
+
+		ret = row_is_in_frame(winstate, current_pos, slot);
+		if (ret <= 0)
+		{
+			elog(DEBUG1, "current row is out of frame at: " INT64_FORMAT, current_pos);
+			return false;
+		}
+	}
+	econtext->ecxt_scantuple = slot;
+
+	/* for PREV */
+	if (current_pos > 0)
+	{
+		slot = winstate->prev_slot;
+		if (!window_gettupleslot(winobj, current_pos - 1, slot))
+		{
+			elog(DEBUG1, "previous row is out of partition at: " INT64_FORMAT, current_pos - 1);
+			econtext->ecxt_outertuple = winstate->null_slot;
+		}
+		else
+		{
+			ret = row_is_in_frame(winstate, current_pos - 1, slot);
+			if (ret <= 0)
+			{
+				elog(DEBUG1, "previous row is out of frame at: " INT64_FORMAT, current_pos - 1);
+				econtext->ecxt_outertuple = winstate->null_slot;
+			}
+			else
+			{
+				econtext->ecxt_outertuple = slot;
+			}
+		}
+	}
+	else
+		econtext->ecxt_outertuple = winstate->null_slot;
+
+	/* for NEXT */
+	slot = winstate->next_slot;
+	if (!window_gettupleslot(winobj, current_pos + 1, slot))
+	{
+		elog(DEBUG1, "next row is out of partiton at: " INT64_FORMAT, current_pos + 1);
+		econtext->ecxt_innertuple = winstate->null_slot;
+	}
+	else
+	{
+		ret = row_is_in_frame(winstate, current_pos + 1, slot);
+		if (ret <= 0)
+		{
+			elog(DEBUG1, "next row is out of frame at: " INT64_FORMAT, current_pos + 1);
+			econtext->ecxt_innertuple = winstate->null_slot;
+		}
+		else
+			econtext->ecxt_innertuple = slot;
+	}
+	return true;
+}
diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c
index b87a624fb2..e4cab36ec9 100644
--- a/src/backend/utils/adt/windowfuncs.c
+++ b/src/backend/utils/adt/windowfuncs.c
@@ -13,6 +13,9 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_collation_d.h"
+#include "executor/executor.h"
+#include "nodes/execnodes.h"
 #include "nodes/supportnodes.h"
 #include "utils/builtins.h"
 #include "windowapi.h"
@@ -36,11 +39,19 @@ typedef struct
 	int64		remainder;		/* (total rows) % (bucket num) */
 } ntile_context;
 
+/*
+ * rpr process information.
+ * Used for AFTER MATCH SKIP PAST LAST ROW
+ */
+typedef struct SkipContext
+{
+	int64		pos;	/* last row absolute position */
+} SkipContext;
+
 static bool rank_up(WindowObject winobj);
 static Datum leadlag_common(FunctionCallInfo fcinfo,
 							bool forward, bool withoffset, bool withdefault);
 
-
 /*
  * utility routine for *_rank functions.
  */
@@ -673,7 +684,7 @@ window_last_value(PG_FUNCTION_ARGS)
 	bool		isnull;
 
 	result = WinGetFuncArgInFrame(winobj, 0,
-								  0, WINDOW_SEEK_TAIL, true,
+								  0, WINDOW_SEEK_TAIL, false,
 								  &isnull, NULL);
 	if (isnull)
 		PG_RETURN_NULL();
@@ -713,3 +724,26 @@ window_nth_value(PG_FUNCTION_ARGS)
 
 	PG_RETURN_DATUM(result);
 }
+
+/*
+ * prev
+ * Dummy function to invoke RPR's navigation operator "PREV".
+ * This is *not* a window function.
+ */
+Datum
+window_prev(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
+/*
+ * next
+ * Dummy function to invoke RPR's navigation operation "NEXT".
+ * This is *not* a window function.
+ */
+Datum
+window_next(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+}
+
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6996073989..fa100b2665 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10397,6 +10397,12 @@
 { oid => '3114', descr => 'fetch the Nth row value',
   proname => 'nth_value', prokind => 'w', prorettype => 'anyelement',
   proargtypes => 'anyelement int4', prosrc => 'window_nth_value' },
+{ oid => '6122', descr => 'previous value',
+  proname => 'prev', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_prev' },
+{ oid => '6123', descr => 'next value',
+  proname => 'next', provolatile => 's', prorettype => 'anyelement',
+  proargtypes => 'anyelement', prosrc => 'window_next' },
 
 # functions for range types
 { oid => '3832', descr => 'I/O',
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..4fd3bd1a93 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2519,6 +2519,14 @@ typedef struct WindowAggState
 	int64		groupheadpos;	/* current row's peer group head position */
 	int64		grouptailpos;	/* " " " " tail position (group end+1) */
 
+	/* these fields are used in Row pattern recognition: */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */	
+	List	   *patternVariableList;	/* list of row pattern variables names (list of String) */
+	List	   *patternRegexpList;	/* list of row pattern regular expressions ('+' or ''. list of String) */
+	List	   *defineVariableList;	/* list of row pattern definition variables (list of String) */
+	List	   *defineClauseList;	/* expression for row pattern definition
+									 * search conditions ExprState list */
+
 	MemoryContext partcontext;	/* context for partition-lifespan data */
 	MemoryContext aggcontext;	/* shared context for aggregate working data */
 	MemoryContext curaggcontext;	/* current aggregate's working data */
@@ -2555,6 +2563,16 @@ typedef struct WindowAggState
 	TupleTableSlot *agg_row_slot;
 	TupleTableSlot *temp_slot_1;
 	TupleTableSlot *temp_slot_2;
+
+	/* temporary slots for RPR */
+	TupleTableSlot *prev_slot;	/* PREV row navigation operator */
+	TupleTableSlot *next_slot;	/* NEXT row navigation operator */
+	TupleTableSlot *null_slot;	/* all NULL slot */
+
+	/* head of the reduced window frame */
+	int64		headpos_in_reduced_frame;
+	/* number of rows in the reduced window frame */
+	int64		num_rows_in_reduced_frame;
 } WindowAggState;
 
 /* ----------------
diff --git a/src/include/windowapi.h b/src/include/windowapi.h
index b8c2c565d1..1e292648e9 100644
--- a/src/include/windowapi.h
+++ b/src/include/windowapi.h
@@ -58,7 +58,15 @@ extern Datum WinGetFuncArgInFrame(WindowObject winobj, int argno,
 								  int relpos, int seektype, bool set_mark,
 								  bool *isnull, bool *isout);
 
+extern int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot,
+							 int relpos, int seektype, bool set_mark,
+							 bool *isnull, bool *isout);
+
 extern Datum WinGetFuncArgCurrent(WindowObject winobj, int argno,
 								  bool *isnull);
 
+extern WindowAggState *WinGetAggState(WindowObject winobj);
+
+extern bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot);
+
 #endif							/* WINDOWAPI_H */
-- 
2.25.1


----Next_Part(Wed_Jul_26_21_21_34_2023_317)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v3-0005-Row-pattern-recognition-patch-docs.patch"



^ permalink  raw  reply  [nested|flat] 108+ messages in thread