public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/3] Avoid GIN full scan for empty ALL keys
33+ messages / 7 participants
[nested] [flat]
* [PATCH 1/3] Avoid GIN full scan for empty ALL keys
@ 2019-08-01 19:59 Nikita Glukhov <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nikita Glukhov @ 2019-08-01 19:59 UTC (permalink / raw)
---
contrib/pg_trgm/expected/pg_trgm.out | 62 ++++++++++++++++++++++++++++++++++++
contrib/pg_trgm/sql/pg_trgm.sql | 16 ++++++++++
src/backend/access/gin/ginget.c | 7 +++-
src/backend/access/gin/ginscan.c | 15 ++++++---
src/backend/utils/adt/selfuncs.c | 12 ++++++-
src/include/access/gin_private.h | 1 +
src/test/regress/expected/gin.out | 30 ++++++++++++++++-
src/test/regress/sql/gin.sql | 14 +++++++-
8 files changed, 149 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_trgm/expected/pg_trgm.out b/contrib/pg_trgm/expected/pg_trgm.out
index b3e709f..3e5ba9b 100644
--- a/contrib/pg_trgm/expected/pg_trgm.out
+++ b/contrib/pg_trgm/expected/pg_trgm.out
@@ -3498,6 +3498,68 @@ select count(*) from test_trgm where t ~ '[qwerty]{2}-?[qwerty]{2}';
1000
(1 row)
+-- check handling of indexquals that generate no searchable conditions
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+ QUERY PLAN
+-----------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+ -> Bitmap Index Scan on trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+(5 rows)
+
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+ count
+-------
+ 19
+(1 row)
+
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+ QUERY PLAN
+-------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qw%'::text))
+ -> Bitmap Index Scan on trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qw%'::text))
+(5 rows)
+
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+ count
+-------
+ 19
+(1 row)
+
+-- ensure that pending-list items are handled correctly, too
+create temp table t_test_trgm(t text COLLATE "C");
+create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops);
+insert into t_test_trgm values ('qwerty99'), ('qwerty01');
+explain (costs off)
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+ QUERY PLAN
+-----------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on t_test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+ -> Bitmap Index Scan on t_trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+(5 rows)
+
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+ count
+-------
+ 1
+(1 row)
+
+select count(*) from t_test_trgm where t like '%99%' and t like '%qw%';
+ count
+-------
+ 1
+(1 row)
+
create table test2(t text COLLATE "C");
insert into test2 values ('abcdef');
insert into test2 values ('quark');
diff --git a/contrib/pg_trgm/sql/pg_trgm.sql b/contrib/pg_trgm/sql/pg_trgm.sql
index 08459e6..dcfd3c2 100644
--- a/contrib/pg_trgm/sql/pg_trgm.sql
+++ b/contrib/pg_trgm/sql/pg_trgm.sql
@@ -55,6 +55,22 @@ select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu098
select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
select count(*) from test_trgm where t ~ '[qwerty]{2}-?[qwerty]{2}';
+-- check handling of indexquals that generate no searchable conditions
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+-- ensure that pending-list items are handled correctly, too
+create temp table t_test_trgm(t text COLLATE "C");
+create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops);
+insert into t_test_trgm values ('qwerty99'), ('qwerty01');
+explain (costs off)
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from t_test_trgm where t like '%99%' and t like '%qw%';
+
create table test2(t text COLLATE "C");
insert into test2 values ('abcdef');
insert into test2 values ('quark');
diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c
index b18ae2b..65ed8b2 100644
--- a/src/backend/access/gin/ginget.c
+++ b/src/backend/access/gin/ginget.c
@@ -1814,7 +1814,7 @@ scanPendingInsert(IndexScanDesc scan, TIDBitmap *tbm, int64 *ntids)
* consistent functions.
*/
oldCtx = MemoryContextSwitchTo(so->tempCtx);
- recheck = false;
+ recheck = so->forcedRecheck;
match = true;
for (i = 0; i < so->nkeys; i++)
@@ -1888,9 +1888,14 @@ gingetbitmap(IndexScanDesc scan, TIDBitmap *tbm)
{
CHECK_FOR_INTERRUPTS();
+ /* Get next item ... */
if (!scanGetItem(scan, iptr, &iptr, &recheck))
break;
+ /* ... apply forced recheck if required ... */
+ recheck |= so->forcedRecheck;
+
+ /* ... and transfer it into bitmap */
if (ItemPointerIsLossyPage(&iptr))
tbm_add_page(tbm, ItemPointerGetBlockNumber(&iptr));
else
diff --git a/src/backend/access/gin/ginscan.c b/src/backend/access/gin/ginscan.c
index 74d9821..11e7e8e 100644
--- a/src/backend/access/gin/ginscan.c
+++ b/src/backend/access/gin/ginscan.c
@@ -286,6 +286,7 @@ ginNewScanKey(IndexScanDesc scan)
palloc(so->allocentries * sizeof(GinScanEntry));
so->isVoidRes = false;
+ so->forcedRecheck = false;
for (i = 0; i < scan->numberOfKeys; i++)
{
@@ -329,10 +330,6 @@ ginNewScanKey(IndexScanDesc scan)
searchMode > GIN_SEARCH_MODE_ALL)
searchMode = GIN_SEARCH_MODE_ALL;
- /* Non-default modes require the index to have placeholders */
- if (searchMode != GIN_SEARCH_MODE_DEFAULT)
- hasNullQuery = true;
-
/*
* In default mode, no keys means an unsatisfiable query.
*/
@@ -343,9 +340,19 @@ ginNewScanKey(IndexScanDesc scan)
so->isVoidRes = true;
break;
}
+ else if (searchMode == GIN_SEARCH_MODE_ALL)
+ {
+ so->forcedRecheck = true;
+ continue;
+ }
+
nQueryValues = 0; /* ensure sane value */
}
+ /* Non-default modes require the index to have placeholders */
+ if (searchMode != GIN_SEARCH_MODE_DEFAULT)
+ hasNullQuery = true;
+
/*
* Create GinNullCategory representation. If the extractQueryFn
* didn't create a nullFlags array, we assume everything is non-null.
diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 7eba59e..1a9d76d 100644
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -6326,6 +6326,16 @@ gincost_pattern(IndexOptInfo *index, int indexcol,
return false;
}
+ if (nentries <= 0 && searchMode == GIN_SEARCH_MODE_ALL)
+ {
+ /*
+ * GIN does not emit scan entries for empty GIN_SEARCH_MODE_ALL keys,
+ * and it can avoid full index scan if there are entries from other
+ * keys, so we can skip setting of 'haveFullScan' flag.
+ */
+ return true;
+ }
+
for (i = 0; i < nentries; i++)
{
/*
@@ -6709,7 +6719,7 @@ gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
return;
}
- if (counts.haveFullScan || indexQuals == NIL)
+ if (counts.haveFullScan || indexQuals == NIL || counts.searchEntries <= 0)
{
/*
* Full index scan will be required. We treat this as if every key in
diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
index afb3e15..b0251f7 100644
--- a/src/include/access/gin_private.h
+++ b/src/include/access/gin_private.h
@@ -359,6 +359,7 @@ typedef struct GinScanOpaqueData
MemoryContext keyCtx; /* used to hold key and entry data */
bool isVoidRes; /* true if query is unsatisfiable */
+ bool forcedRecheck; /* must recheck all returned tuples */
} GinScanOpaqueData;
typedef GinScanOpaqueData *GinScanOpaque;
diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out
index a3911a6..fb0d29c 100644
--- a/src/test/regress/expected/gin.out
+++ b/src/test/regress/expected/gin.out
@@ -1,7 +1,7 @@
--
-- Test GIN indexes.
--
--- There are other tests to test different GIN opclassed. This is for testing
+-- There are other tests to test different GIN opclasses. This is for testing
-- GIN itself.
-- Create and populate a test table with a GIN index.
create table gin_test_tbl(i int4[]) with (autovacuum_enabled = off);
@@ -35,3 +35,31 @@ insert into gin_test_tbl select array[1, 2, g] from generate_series(1, 1000) g;
insert into gin_test_tbl select array[1, 3, g] from generate_series(1, 1000) g;
delete from gin_test_tbl where i @> array[2];
vacuum gin_test_tbl;
+-- Test optimization of empty queries
+create temp table t_gin_test_tbl(i int4[], j int4[]);
+create index on t_gin_test_tbl using gin (i, j);
+insert into t_gin_test_tbl select array[100,g], array[200,g]
+from generate_series(1, 10) g;
+insert into t_gin_test_tbl values(array[0,0], null);
+set enable_seqscan = off;
+explain
+select * from t_gin_test_tbl where array[0] <@ i;
+ QUERY PLAN
+--------------------------------------------------------------------------------------
+ Bitmap Heap Scan on t_gin_test_tbl (cost=12.03..20.49 rows=4 width=64)
+ Recheck Cond: ('{0}'::integer[] <@ i)
+ -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx (cost=0.00..12.03 rows=4 width=0)
+ Index Cond: (i @> '{0}'::integer[])
+(4 rows)
+
+select * from t_gin_test_tbl where array[0] <@ i;
+ i | j
+-------+---
+ {0,0} |
+(1 row)
+
+select * from t_gin_test_tbl where array[0] <@ i and '{}'::int4[] <@ j;
+ i | j
+---+---
+(0 rows)
+
diff --git a/src/test/regress/sql/gin.sql b/src/test/regress/sql/gin.sql
index c566e9b..aaf9c19 100644
--- a/src/test/regress/sql/gin.sql
+++ b/src/test/regress/sql/gin.sql
@@ -1,7 +1,7 @@
--
-- Test GIN indexes.
--
--- There are other tests to test different GIN opclassed. This is for testing
+-- There are other tests to test different GIN opclasses. This is for testing
-- GIN itself.
-- Create and populate a test table with a GIN index.
@@ -34,3 +34,15 @@ insert into gin_test_tbl select array[1, 3, g] from generate_series(1, 1000) g;
delete from gin_test_tbl where i @> array[2];
vacuum gin_test_tbl;
+
+-- Test optimization of empty queries
+create temp table t_gin_test_tbl(i int4[], j int4[]);
+create index on t_gin_test_tbl using gin (i, j);
+insert into t_gin_test_tbl select array[100,g], array[200,g]
+from generate_series(1, 10) g;
+insert into t_gin_test_tbl values(array[0,0], null);
+set enable_seqscan = off;
+explain
+select * from t_gin_test_tbl where array[0] <@ i;
+select * from t_gin_test_tbl where array[0] <@ i;
+select * from t_gin_test_tbl where array[0] <@ i and '{}'::int4[] <@ j;
--
2.7.4
--------------B7A985DD4CDB1298897DF1B4
Content-Type: text/x-patch;
name="0002-Force-GIN-recheck-more-accurately-v06.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002-Force-GIN-recheck-more-accurately-v06.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 1/5] Avoid GIN full scan for empty ALL keys
@ 2019-11-15 14:15 Nikita Glukhov <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nikita Glukhov @ 2019-11-15 14:15 UTC (permalink / raw)
---
contrib/pg_trgm/expected/pg_trgm.out | 62 ++++++++++++++++++++++++++++++++++++
contrib/pg_trgm/sql/pg_trgm.sql | 16 ++++++++++
src/backend/access/gin/ginget.c | 7 +++-
src/backend/access/gin/ginscan.c | 19 +++++++++--
src/backend/utils/adt/selfuncs.c | 28 ++++++++++++++--
src/include/access/gin_private.h | 1 +
src/test/regress/expected/gin.out | 31 +++++++++++++++++-
src/test/regress/sql/gin.sql | 15 ++++++++-
8 files changed, 170 insertions(+), 9 deletions(-)
diff --git a/contrib/pg_trgm/expected/pg_trgm.out b/contrib/pg_trgm/expected/pg_trgm.out
index b3e709f..3e5ba9b 100644
--- a/contrib/pg_trgm/expected/pg_trgm.out
+++ b/contrib/pg_trgm/expected/pg_trgm.out
@@ -3498,6 +3498,68 @@ select count(*) from test_trgm where t ~ '[qwerty]{2}-?[qwerty]{2}';
1000
(1 row)
+-- check handling of indexquals that generate no searchable conditions
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+ QUERY PLAN
+-----------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+ -> Bitmap Index Scan on trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+(5 rows)
+
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+ count
+-------
+ 19
+(1 row)
+
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+ QUERY PLAN
+-------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qw%'::text))
+ -> Bitmap Index Scan on trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qw%'::text))
+(5 rows)
+
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+ count
+-------
+ 19
+(1 row)
+
+-- ensure that pending-list items are handled correctly, too
+create temp table t_test_trgm(t text COLLATE "C");
+create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops);
+insert into t_test_trgm values ('qwerty99'), ('qwerty01');
+explain (costs off)
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+ QUERY PLAN
+-----------------------------------------------------------------------------
+ Aggregate
+ -> Bitmap Heap Scan on t_test_trgm
+ Recheck Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+ -> Bitmap Index Scan on t_trgm_idx
+ Index Cond: ((t ~~ '%99%'::text) AND (t ~~ '%qwerty%'::text))
+(5 rows)
+
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+ count
+-------
+ 1
+(1 row)
+
+select count(*) from t_test_trgm where t like '%99%' and t like '%qw%';
+ count
+-------
+ 1
+(1 row)
+
create table test2(t text COLLATE "C");
insert into test2 values ('abcdef');
insert into test2 values ('quark');
diff --git a/contrib/pg_trgm/sql/pg_trgm.sql b/contrib/pg_trgm/sql/pg_trgm.sql
index 08459e6..dcfd3c2 100644
--- a/contrib/pg_trgm/sql/pg_trgm.sql
+++ b/contrib/pg_trgm/sql/pg_trgm.sql
@@ -55,6 +55,22 @@ select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu098
select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
select count(*) from test_trgm where t ~ '[qwerty]{2}-?[qwerty]{2}';
+-- check handling of indexquals that generate no searchable conditions
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from test_trgm where t like '%99%' and t like '%qwerty%';
+explain (costs off)
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+select count(*) from test_trgm where t like '%99%' and t like '%qw%';
+-- ensure that pending-list items are handled correctly, too
+create temp table t_test_trgm(t text COLLATE "C");
+create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops);
+insert into t_test_trgm values ('qwerty99'), ('qwerty01');
+explain (costs off)
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%';
+select count(*) from t_test_trgm where t like '%99%' and t like '%qw%';
+
create table test2(t text COLLATE "C");
insert into test2 values ('abcdef');
insert into test2 values ('quark');
diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c
index b18ae2b..65ed8b2 100644
--- a/src/backend/access/gin/ginget.c
+++ b/src/backend/access/gin/ginget.c
@@ -1814,7 +1814,7 @@ scanPendingInsert(IndexScanDesc scan, TIDBitmap *tbm, int64 *ntids)
* consistent functions.
*/
oldCtx = MemoryContextSwitchTo(so->tempCtx);
- recheck = false;
+ recheck = so->forcedRecheck;
match = true;
for (i = 0; i < so->nkeys; i++)
@@ -1888,9 +1888,14 @@ gingetbitmap(IndexScanDesc scan, TIDBitmap *tbm)
{
CHECK_FOR_INTERRUPTS();
+ /* Get next item ... */
if (!scanGetItem(scan, iptr, &iptr, &recheck))
break;
+ /* ... apply forced recheck if required ... */
+ recheck |= so->forcedRecheck;
+
+ /* ... and transfer it into bitmap */
if (ItemPointerIsLossyPage(&iptr))
tbm_add_page(tbm, ItemPointerGetBlockNumber(&iptr));
else
diff --git a/src/backend/access/gin/ginscan.c b/src/backend/access/gin/ginscan.c
index 74d9821..7b8de10 100644
--- a/src/backend/access/gin/ginscan.c
+++ b/src/backend/access/gin/ginscan.c
@@ -286,6 +286,7 @@ ginNewScanKey(IndexScanDesc scan)
palloc(so->allocentries * sizeof(GinScanEntry));
so->isVoidRes = false;
+ so->forcedRecheck = false;
for (i = 0; i < scan->numberOfKeys; i++)
{
@@ -333,16 +334,28 @@ ginNewScanKey(IndexScanDesc scan)
if (searchMode != GIN_SEARCH_MODE_DEFAULT)
hasNullQuery = true;
- /*
- * In default mode, no keys means an unsatisfiable query.
- */
+ /* Special cases for queries that contain no keys */
if (queryValues == NULL || nQueryValues <= 0)
{
if (searchMode == GIN_SEARCH_MODE_DEFAULT)
{
+ /* In default mode, no keys means an unsatisfiable query */
so->isVoidRes = true;
break;
}
+ else if (searchMode == GIN_SEARCH_MODE_ALL)
+ {
+ /*
+ * The query probably matches all non-null items, but rather
+ * than scanning the index in ALL mode, we use forced rechecks
+ * to verify matches of this scankey. This wins if there are
+ * any non-ALL scankeys; otherwise we end up adding an
+ * EVERYTHING scankey below.
+ */
+ so->forcedRecheck = true;
+ continue;
+ }
+
nQueryValues = 0; /* ensure sane value */
}
diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 26a2e3b..46fff24 100644
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -6316,10 +6316,24 @@ gincost_pattern(IndexOptInfo *index, int indexcol,
PointerGetDatum(&nullFlags),
PointerGetDatum(&searchMode));
- if (nentries <= 0 && searchMode == GIN_SEARCH_MODE_DEFAULT)
+ /* Special cases for queries that contain no keys */
+ if (nentries <= 0)
{
- /* No match is possible */
- return false;
+ if (searchMode == GIN_SEARCH_MODE_DEFAULT)
+ {
+ /* In default mode, no keys means an unsatisfiable query */
+ return false;
+ }
+ else if (searchMode == GIN_SEARCH_MODE_ALL)
+ {
+ /*
+ * ginNewScanKey() does not emit scankeys for a key-less ALL
+ * query. Instead it will emit an EVERYTHING key, but only if
+ * there are no other regular keys. We don't know that yet, so
+ * postpone setting the haveFullScan flag.
+ */
+ return true;
+ }
}
for (i = 0; i < nentries; i++)
@@ -6481,6 +6495,10 @@ gincost_scalararrayopexpr(PlannerInfo *root,
/* We ignore array elements that are unsatisfiable patterns */
numPossible++;
+ /* If no regular scan keys, assume an EVERYTHING scan is needed */
+ if (elemcounts.searchEntries == 0)
+ elemcounts.haveFullScan = true;
+
if (elemcounts.haveFullScan)
{
/*
@@ -6705,6 +6723,10 @@ gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
return;
}
+ /* If no regular scan keys, assume an EVERYTHING scan is needed */
+ if (counts.searchEntries == 0)
+ counts.haveFullScan = true;
+
if (counts.haveFullScan || indexQuals == NIL)
{
/*
diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
index 78fcd82..9d2ee3a 100644
--- a/src/include/access/gin_private.h
+++ b/src/include/access/gin_private.h
@@ -359,6 +359,7 @@ typedef struct GinScanOpaqueData
MemoryContext keyCtx; /* used to hold key and entry data */
bool isVoidRes; /* true if query is unsatisfiable */
+ bool forcedRecheck; /* must recheck all returned tuples */
} GinScanOpaqueData;
typedef GinScanOpaqueData *GinScanOpaque;
diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out
index a3911a6..5ba96fa 100644
--- a/src/test/regress/expected/gin.out
+++ b/src/test/regress/expected/gin.out
@@ -1,7 +1,7 @@
--
-- Test GIN indexes.
--
--- There are other tests to test different GIN opclassed. This is for testing
+-- There are other tests to test different GIN opclasses. This is for testing
-- GIN itself.
-- Create and populate a test table with a GIN index.
create table gin_test_tbl(i int4[]) with (autovacuum_enabled = off);
@@ -35,3 +35,32 @@ insert into gin_test_tbl select array[1, 2, g] from generate_series(1, 1000) g;
insert into gin_test_tbl select array[1, 3, g] from generate_series(1, 1000) g;
delete from gin_test_tbl where i @> array[2];
vacuum gin_test_tbl;
+-- Test optimization of empty queries
+create temp table t_gin_test_tbl(i int4[], j int4[]);
+create index on t_gin_test_tbl using gin (i, j);
+insert into t_gin_test_tbl select array[100,g], array[200,g]
+from generate_series(1, 10) g;
+insert into t_gin_test_tbl values(array[0,0], null);
+set enable_seqscan = off;
+explain (costs off)
+select * from t_gin_test_tbl where array[0] <@ i;
+ QUERY PLAN
+---------------------------------------------------
+ Bitmap Heap Scan on t_gin_test_tbl
+ Recheck Cond: ('{0}'::integer[] <@ i)
+ -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx
+ Index Cond: (i @> '{0}'::integer[])
+(4 rows)
+
+select * from t_gin_test_tbl where array[0] <@ i;
+ i | j
+-------+---
+ {0,0} |
+(1 row)
+
+select * from t_gin_test_tbl where array[0] <@ i and '{}'::int4[] <@ j;
+ i | j
+---+---
+(0 rows)
+
+reset enable_seqscan;
diff --git a/src/test/regress/sql/gin.sql b/src/test/regress/sql/gin.sql
index c566e9b..f98fb7e 100644
--- a/src/test/regress/sql/gin.sql
+++ b/src/test/regress/sql/gin.sql
@@ -1,7 +1,7 @@
--
-- Test GIN indexes.
--
--- There are other tests to test different GIN opclassed. This is for testing
+-- There are other tests to test different GIN opclasses. This is for testing
-- GIN itself.
-- Create and populate a test table with a GIN index.
@@ -34,3 +34,16 @@ insert into gin_test_tbl select array[1, 3, g] from generate_series(1, 1000) g;
delete from gin_test_tbl where i @> array[2];
vacuum gin_test_tbl;
+
+-- Test optimization of empty queries
+create temp table t_gin_test_tbl(i int4[], j int4[]);
+create index on t_gin_test_tbl using gin (i, j);
+insert into t_gin_test_tbl select array[100,g], array[200,g]
+from generate_series(1, 10) g;
+insert into t_gin_test_tbl values(array[0,0], null);
+set enable_seqscan = off;
+explain (costs off)
+select * from t_gin_test_tbl where array[0] <@ i;
+select * from t_gin_test_tbl where array[0] <@ i;
+select * from t_gin_test_tbl where array[0] <@ i and '{}'::int4[] <@ j;
+reset enable_seqscan;
--
2.7.4
--------------C5A6759166AE2CA1CB41B4D8
Content-Type: text/x-patch;
name="0002-Avoid-GIN-recheck-for-NULLs-using-new-search-mode-v08.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="0002-Avoid-GIN-recheck-for-NULLs-using-new-search-mode-v08.p";
filename*1="atch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v24 08/15] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 299 ++++++++-
src/backend/commands/matview.c | 1016 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1281 insertions(+), 35 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 91888891bf..ce6e6ffe33 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,9 +100,10 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query =
*qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static void CreateIndexOnIMMV(Query *query, Relation matviewRel);
static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -432,6 +439,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -446,14 +454,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -470,6 +510,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -923,11 +1048,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1008,6 +1135,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1056,7 +1185,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1067,8 +1196,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1080,15 +1213,128 @@ check_ivm_restriction_walker(Node *node, void *con=
text)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ case T_Aggref:
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
+ default:
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
}
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1138,7 +1384,30 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (qry->distinctClause)
+
+ if (qry->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, qry->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, qry->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (qry->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, qry->targetList)
@@ -1197,7 +1466,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 70e35e5a63..8a06b4e799 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -81,6 +81,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -117,8 +143,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -152,7 +186,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -163,19 +197,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1440,8 +1503,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1492,7 +1555,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1854,17 +1917,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1933,6 +2013,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1946,11 +2028,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1968,6 +2055,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -1991,7 +2087,65 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, resname);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2005,6 +2159,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2022,10 +2178,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2048,7 +2213,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2063,49 +2228,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
+ "%s" /* RETURNING clause for recalc infomation */
"), dlt AS (" /* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
- ")",
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2165,10 +2691,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2199,6 +2730,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2206,6 +2738,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2280,6 +2813,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2313,6 +3189,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2321,6 +3204,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index a57ce463e1..702b097079 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -29,6 +29,7 @@ extern ObjectAddress ExecCreateTableAs(ParseState *pstate=
, CreateTableAsStmt *st
extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid matviewOid, bool=
is_create);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Thu__23_Sep_2021_04_57_30_+0900_b5pmgR1N8oMaMz.U
Content-Type: text/x-diff;
name="v24-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Disposition: attachment;
filename="v24-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v24 08/15] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 299 ++++++++-
src/backend/commands/matview.c | 1016 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1281 insertions(+), 35 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 91888891bf..ce6e6ffe33 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,9 +100,10 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query =
*qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static void CreateIndexOnIMMV(Query *query, Relation matviewRel);
static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -432,6 +439,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -446,14 +454,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -470,6 +510,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -923,11 +1048,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1008,6 +1135,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1056,7 +1185,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1067,8 +1196,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1080,15 +1213,128 @@ check_ivm_restriction_walker(Node *node, void *con=
text)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ case T_Aggref:
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
+ default:
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
}
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1138,7 +1384,30 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (qry->distinctClause)
+
+ if (qry->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, qry->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, qry->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (qry->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, qry->targetList)
@@ -1197,7 +1466,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index a27d23434d..219444571a 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -81,6 +81,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -117,8 +143,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -152,7 +186,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -163,19 +197,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1441,8 +1504,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1493,7 +1556,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1855,17 +1918,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1934,6 +2014,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1947,11 +2029,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1969,6 +2056,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -1992,7 +2088,65 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, resname);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2006,6 +2160,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2023,10 +2179,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2049,7 +2214,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2064,49 +2229,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
+ "%s" /* RETURNING clause for recalc infomation */
"), dlt AS (" /* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
- ")",
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2166,10 +2692,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2200,6 +2731,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2207,6 +2739,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2281,6 +2814,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2314,6 +3190,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2322,6 +3205,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index a57ce463e1..702b097079 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -29,6 +29,7 @@ extern ObjectAddress ExecCreateTableAs(ParseState *pstate=
, CreateTableAsStmt *st
extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid matviewOid, bool=
is_create);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Fri__29_Oct_2021_18_16_28_+0900_jlYRKjywLqhZ7oyk
Content-Type: text/x-diff;
name="v24-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Disposition: attachment;
filename="v24-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v26 08/10] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 296 ++++++++-
src/backend/commands/matview.c | 1016 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1278 insertions(+), 35 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 1fbcede7aa..83a1b3de9e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,9 +100,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
-static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList, bool is_create);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList, bool is_create);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -429,6 +435,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -443,14 +450,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -467,6 +506,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -920,11 +1044,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1005,6 +1131,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1053,7 +1181,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1064,8 +1192,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1077,9 +1209,36 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
+ }
+ case T_Aggref:
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
}
- break;
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1087,6 +1246,91 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1144,7 +1388,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel,=
bool is_create)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1202,7 +1468,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel, =
bool is_create)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 29cf18360e..3721774a9b 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -81,6 +81,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -117,8 +143,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -152,7 +186,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -163,19 +197,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1447,8 +1510,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1499,7 +1562,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1861,17 +1924,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1940,6 +2020,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1953,11 +2035,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1975,6 +2062,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -1998,7 +2094,65 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, resname);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2012,6 +2166,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2029,10 +2185,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2055,7 +2220,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2070,49 +2235,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
+ "%s" /* RETURNING clause for recalc infomation */
"), dlt AS (" /* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
- ")",
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2172,10 +2698,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2206,6 +2737,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2213,6 +2745,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2287,6 +2820,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2320,6 +3196,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2328,6 +3211,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index c369b3ba5e..abcc31023c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid, bool is_cr
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel, bool is_c=
reate);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Mon__14_Mar_2022_19_12_17_+0900_E9HVp8j5QFkIIek.
Content-Type: text/x-diff;
name="v26-0007-Add-Incremental-View-Maintenance-support.patch"
Content-Disposition: attachment;
filename="v26-0007-Add-Incremental-View-Maintenance-support.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v27 7/9] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 299 ++++++++-
src/backend/commands/matview.c | 1023 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1281 insertions(+), 42 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 1224a3b075..d2ae50d5ee 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,9 +100,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
-static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList, bool is_create);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList, bool is_create);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -429,6 +435,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -443,14 +450,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -467,6 +506,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -920,11 +1044,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1005,6 +1131,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1053,7 +1181,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1064,8 +1192,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1077,14 +1209,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1092,6 +1246,91 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1149,7 +1388,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel,=
bool is_create)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1207,7 +1468,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel, =
bool is_create)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index cb713328a0..721d91c009 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -82,6 +82,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -118,8 +144,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -153,7 +187,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -164,19 +198,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1470,8 +1533,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1522,7 +1585,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1884,17 +1947,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1963,6 +2043,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1976,11 +2058,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1998,6 +2085,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2014,13 +2110,72 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2034,6 +2189,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2051,10 +2208,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2077,7 +2243,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2092,49 +2258,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
- ")"
- /* delete a tuple if this is to be deleted */
- "DELETE FROM %s AS mv USING t "
- "WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
+ "%s" /* RETURNING clause for recalc infomation */
+ "), dlt AS (" /* delete a tuple if this is to be deleted */
+ "DELETE FROM %s AS mv USING t "
+ "WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2194,10 +2721,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2228,6 +2760,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2235,6 +2768,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2309,6 +2843,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2342,6 +3219,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2350,6 +3234,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index c369b3ba5e..abcc31023c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid, bool is_cr
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel, bool is_c=
reate);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Fri__22_Apr_2022_11_29_39_+0900_ZOAC7UMt5e8j1Nvx
Content-Type: text/x-diff;
name="v27-0008-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Disposition: attachment;
filename="v27-0008-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v23 08/15] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group keys. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except to count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 294 ++++++++-
src/backend/commands/matview.c | 1016 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1280 insertions(+), 31 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 3fc2af1c4a..aebbb7290e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -93,9 +99,10 @@ static void intorel_destroy(DestReceiver *self);
static void CreateIvmTriggersOnBaseTables_recurse(Query *qry, Node *node, =
Oid matviewOid, Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static void CreateIndexOnIMMV(Query *query, Relation matviewRel);
static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -431,6 +438,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -445,12 +453,45 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
+ {
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appeared in select list is not supp=
orted on incrementally maintainable materialized view")));
+ }
+ }
/* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
- if (rewritten->distinctClause)
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
- if (rewritten->distinctClause)
+ if (rewritten->distinctClause || rewritten->hasAggs)
{
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
@@ -468,6 +509,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except to count, add count() func with the sam=
e arg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -923,11 +1049,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
/* This can recurse, so check for excessive recursion */
check_stack_depth();
@@ -1008,6 +1136,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1041,7 +1171,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1052,8 +1182,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1065,15 +1199,128 @@ check_ivm_restriction_walker(Node *node, void *con=
text)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ case T_Aggref:
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
+ default:
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
}
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateindexOnIMMV
*
@@ -1123,7 +1370,32 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (qry->distinctClause)
+
+ if (qry->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, qry->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, qry->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+
+ index->isconstraint =3D true;
+ }
+ else if (qry->distinctClause)
{
/* create unique constraint on all columns */
index->isconstraint =3D true;
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index f1f46e2e14..bce2e3ae3d 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -81,6 +81,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -117,8 +143,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -152,7 +186,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -163,19 +197,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1432,8 +1495,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1484,7 +1547,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1846,17 +1909,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1925,6 +2005,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1938,11 +2020,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1960,6 +2047,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -1983,7 +2079,65 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, resname);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -1997,6 +2151,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2014,10 +2170,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2040,7 +2205,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2055,49 +2220,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
+ "%s" /* RETURNING clause for recalc infomation */
"), dlt AS (" /* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
- ")",
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2157,10 +2683,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2191,6 +2722,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2198,6 +2730,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2272,6 +2805,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2305,6 +3181,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2313,6 +3196,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index a57ce463e1..702b097079 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -29,6 +29,7 @@ extern ObjectAddress ExecCreateTableAs(ParseState *pstate=
, CreateTableAsStmt *st
extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid matviewOid, bool=
is_create);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Mon__2_Aug_2021_15_28_34_+0900_wlHCjIpnD/FrGAKu
Content-Type: text/x-diff;
name="v23-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Disposition: attachment;
filename="v23-0009-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v27 7/9] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 299 ++++++++-
src/backend/commands/matview.c | 1023 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1281 insertions(+), 42 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 1224a3b075..d2ae50d5ee 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,9 +100,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
-static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList, bool is_create);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList, bool is_create);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -429,6 +435,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -443,14 +450,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -467,6 +506,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -920,11 +1044,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1005,6 +1131,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1053,7 +1181,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1064,8 +1192,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1077,14 +1209,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1092,6 +1246,91 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1149,7 +1388,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel,=
bool is_create)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1207,7 +1468,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel, =
bool is_create)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index cb713328a0..721d91c009 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -82,6 +82,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -118,8 +144,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -153,7 +187,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -164,19 +198,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1470,8 +1533,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1522,7 +1585,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1884,17 +1947,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1963,6 +2043,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1976,11 +2058,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1998,6 +2085,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2014,13 +2110,72 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2034,6 +2189,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2051,10 +2208,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2077,7 +2243,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2092,49 +2258,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
- ")"
- /* delete a tuple if this is to be deleted */
- "DELETE FROM %s AS mv USING t "
- "WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
+ "%s" /* RETURNING clause for recalc infomation */
+ "), dlt AS (" /* delete a tuple if this is to be deleted */
+ "DELETE FROM %s AS mv USING t "
+ "WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2194,10 +2721,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2228,6 +2760,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2235,6 +2768,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2309,6 +2843,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2342,6 +3219,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2350,6 +3234,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index c369b3ba5e..abcc31023c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid, bool is_cr
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel, bool is_c=
reate);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Fri__22_Apr_2022_14_58_01_+0900_MN3L/o2YUVF2g4zw
Content-Type: text/x-diff;
name="v27-0008-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Disposition: attachment;
filename="v27-0008-Add-regression-tests-for-Incremental-View-Mainte.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v25 08/15] Add aggregates support in IVM
@ 2021-08-02 05:59 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2021-08-02 05:59 UTC (permalink / raw)
Currently, count, sum, avg, min and max are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
In the case of views without aggregate functions, only the number of
tuple multiplicities in __ivm_count__ column are updated at incremental
maintenance. On the other hand, in the case of view with aggregates,
the aggregated values and related hidden columns are also updated. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
In min or max cases, it becomes more complicated. For an example of min,
when tuples are inserted, the smaller value between the current min value
in the view and the value calculated from the new delta table is used.
When tuples are deleted, if the current min value in the view equals to
the min in the old delta table, we need re-computation the latest min
value from base tables. Otherwise, the current value in the view remains.
As to sum, avg, min, and max (any aggregate functions except count),
NULL in input values is ignored, and this returns a null value when no
rows are selected. To support this specification, the number of not-NULL
input values is counted and stored in views as a hidden column. In the
case of count(), count(x) returns zero when no rows are selected, and
count(*) doesn't ignore NULL input. These specification are also supported.
---
src/backend/commands/createas.c | 299 ++++++++-
src/backend/commands/matview.c | 1016 ++++++++++++++++++++++++++++++-
src/include/commands/createas.h | 1 +
3 files changed, 1281 insertions(+), 35 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 4b73965e56..c58cacec05 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -63,6 +63,7 @@
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
@@ -80,6 +81,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -94,8 +100,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *qry, List **con=
straintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -431,6 +438,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -445,14 +453,46 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
+
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL ? tle->resname : strVal(list_nth=
(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, resname, &next_resno, &a=
ggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
fn->agg_star =3D true;
=20
@@ -469,6 +509,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICI=
T_CALL, -1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(list_make1(makeString("sum")), NIL, COERCE_EXPLICIT_=
CALL, -1);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -922,11 +1047,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -1007,6 +1134,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1055,7 +1184,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1066,8 +1195,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1079,15 +1212,128 @@ check_ivm_restriction_walker(Node *node, void *con=
text)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ case T_Aggref:
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
+ default:
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
}
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ /* min */
+ case F_MIN_ANYARRAY:
+ case F_MIN_INT8:
+ case F_MIN_INT4:
+ case F_MIN_INT2:
+ case F_MIN_OID:
+ case F_MIN_FLOAT4:
+ case F_MIN_FLOAT8:
+ case F_MIN_DATE:
+ case F_MIN_TIME:
+ case F_MIN_TIMETZ:
+ case F_MIN_MONEY:
+ case F_MIN_TIMESTAMP:
+ case F_MIN_TIMESTAMPTZ:
+ case F_MIN_INTERVAL:
+ case F_MIN_TEXT:
+ case F_MIN_NUMERIC:
+ case F_MIN_BPCHAR:
+ case F_MIN_TID:
+ case F_MIN_ANYENUM:
+ case F_MIN_INET:
+ case F_MIN_PG_LSN:
+
+ /* max */
+ case F_MAX_ANYARRAY:
+ case F_MAX_INT8:
+ case F_MAX_INT4:
+ case F_MAX_INT2:
+ case F_MAX_OID:
+ case F_MAX_FLOAT4:
+ case F_MAX_FLOAT8:
+ case F_MAX_DATE:
+ case F_MAX_TIME:
+ case F_MAX_TIMETZ:
+ case F_MAX_MONEY:
+ case F_MAX_TIMESTAMP:
+ case F_MAX_TIMESTAMPTZ:
+ case F_MAX_INTERVAL:
+ case F_MAX_TEXT:
+ case F_MAX_NUMERIC:
+ case F_MAX_BPCHAR:
+ case F_MAX_TID:
+ case F_MAX_ANYENUM:
+ case F_MAX_INET:
+ case F_MAX_PG_LSN:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1139,7 +1385,30 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (qry->distinctClause)
+
+ if (qry->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, qry->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, qry->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (qry->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, qry->targetList)
@@ -1198,7 +1467,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 943de5dfba..1f50aaa1b8 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -81,6 +81,32 @@ typedef struct
=20
#define MV_INIT_QUERYHASHSIZE 16
=20
+/* MV query type codes */
+#define MV_PLAN_RECALC 1
+#define MV_PLAN_SET_VALUE 2
+
+/*
+ * MI_QueryKey
+ *
+ * The key identifying a prepared SPI plan in our query hashtable
+ */
+typedef struct MV_QueryKey
+{
+ Oid matview_id; /* OID of materialized view */
+ int32 query_type; /* query type ID, see MV_PLAN_XXX above */
+} MV_QueryKey;
+
+/*
+ * MV_QueryHashEntry
+ *
+ * Hash entry for cached plans used to maintain materialized views.
+ */
+typedef struct MV_QueryHashEntry
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+} MV_QueryHashEntry;
+
/*
* MV_TriggerHashEntry
*
@@ -117,8 +143,16 @@ typedef struct MV_TriggerTable
RangeTblEntry *original_rte; /* the original RTE saved before rewriting q=
uery */
} MV_TriggerTable;
=20
+static HTAB *mv_query_cache =3D NULL;
static HTAB *mv_trigger_info =3D NULL;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -152,7 +186,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv);
static RangeTblEntry *union_ENRs(RangeTblEntry *rte, Oid relid, List *enr_=
rtes, const char *prefix,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_distinct(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_distinct_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -163,19 +197,48 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static void append_set_clause_for_minmax(const char *resname, StringInfo b=
uf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
+static char *get_returning_string(List *minmax_list, List *is_min_list, Li=
st *keys);
+static char *get_minmax_recalc_condition_string(List *minmax_list, List *i=
s_min_list);
+static char *get_select_for_recalc_string(List *keys);
+static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 nu=
m_tuples,
+ List *namelist, List *keys, Relation matviewRel);
+static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List=
*keys, Oid *keyTypes);
+static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewnam=
e, List *namelist,
+ Oid *valTypes);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
=20
static void mv_InitHashTables(void);
+static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key);
+static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan);
+static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query=
_type);
static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry);
=20
static List *get_securityQuals(Oid relId, int rt_index, Query *query);
@@ -1447,8 +1510,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
entry->xid, entry->cid,
pstate);
- /* Rewrite for DISTINCT clause */
- rewritten =3D rewrite_query_for_distinct(rewritten, pstate);
+ /* Rewrite for DISTINCT clause and aggregates functions */
+ rewritten =3D rewrite_query_for_distinct_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1499,7 +1562,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1861,17 +1924,34 @@ union_ENRs(RangeTblEntry *rte, Oid relid, List *enr=
_rtes, const char *prefix,
}
=20
/*
- * rewrite_query_for_distinct
+ * rewrite_query_for_distinct_and_aggregates
*
- * Rewrite query for counting DISTINCT clause.
+ * Rewrite query for counting DISTINCT clause and aggregate functions.
*/
static Query *
-rewrite_query_for_distinct(Query *query, ParseState *pstate)
+rewrite_query_for_distinct_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(list_make1(makeString("count")), NIL, COERCE_EXPLICIT=
_CALL, -1);
fn->agg_star =3D true;
@@ -1940,6 +2020,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -1953,11 +2035,16 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
int i;
List *keys =3D NIL;
+ List *minmax_list =3D NIL;
+ List *is_min_list =3D NIL;
=20
=20
/*
@@ -1975,6 +2062,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -1998,7 +2094,65 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, resname);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ /* min/max */
+ else if (!strcmp(aggname, "min") || !strcmp(aggname, "max"))
+ {
+ bool is_min =3D (!strcmp(aggname, "min"));
+
+ append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs=
_list_buf, is_min);
+
+ /* make a resname list of min and max aggregates */
+ minmax_list =3D lappend(minmax_list, resname);
+ is_min_list =3D lappend_int(is_min_list, is_min);
+ }
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2012,6 +2166,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
{
EphemeralNamedRelation enr =3D palloc(sizeof(EphemeralNamedRelationData)=
);
+ SPITupleTable *tuptable_recalc =3D NULL;
+ uint64 num_recalc;
int rc;
=20
/* convert tuplestores to ENR, and register for SPI */
@@ -2029,10 +2185,19 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ minmax_list, is_min_list,
+ count_colname, &tuptable_recalc, &num_recalc);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
+ /*
+ * If we have min or max, we might have to recalculate aggregate values =
from base tables
+ * on some tuples. TIDs and keys such tuples are returned as a result of=
the above query.
+ */
+ if (minmax_list && tuptable_recalc)
+ recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, m=
atviewRel);
+
}
/* For tuple insertion */
if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
@@ -2055,7 +2220,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2070,49 +2235,410 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_t=
uplestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_minmax
+ *
+ * Append SET clause string for min or max aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ * is_min is true if this is min, false if not.
+ */
+static void
+append_set_clause_for_minmax(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ bool is_min)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /*
+ * If the new value doesn't became NULL then use the value remaining
+ * in the view although this will be recomputated afterwords.
+ */
+ appendStringInfo(buf_old,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_SUB, "mv", "t", count_col),
+ quote_qualified_identifier("mv", resname)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /*
+ * min =3D LEAST(mv.min, diff.min)
+ * max =3D GREATEST(mv.max, diff.max)
+ */
+ appendStringInfo(buf_new,
+ ", %s =3D CASE WHEN %s THEN NULL ELSE %s(%s,%s) END",
+ quote_qualified_identifier(NULL, resname),
+ get_null_condition_string(IVM_ADD, "mv", "diff", count_col),
+
+ is_min ? "LEAST" : "GREATEST",
+ quote_qualified_identifier("mv", resname),
+ quote_qualified_identifier("diff", resname)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
* Execute a query for applying a delta table given by deltname_old
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
- * has aggregate or distinct.
+ * has aggregate or distinct. Also, when a table in EXISTS sub queries
+ * is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
+ *
+ * If the view has min or max aggregate, this requires a list of resnames =
of
+ * min/max aggregates and a list of boolean which represents which entries=
in
+ * minmax_list is min. These are necessary to check if we need to recalcul=
ate
+ * min or max aggregate values. In this case, this query returns TID and k=
eys
+ * of tuples which need to be recalculated. This result and the number of=
rows
+ * are stored in tuptables and num_recalc repectedly.
+ *
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ List *minmax_list, List *is_min_list,
+ const char *count_colname,
+ SPITupleTable **tuptable_recalc, uint64 *num_recalc)
{
StringInfoData querybuf;
char *match_cond;
+ char *updt_returning =3D "";
+ char *select_for_recalc =3D "SELECT";
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
+
+ Assert(tuptable_recalc !=3D NULL);
+ Assert(num_recalc !=3D NULL);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
=20
+ /*
+ * We need a special RETURNING clause and SELECT statement for min/max to
+ * check which tuple needs re-calculation from base tables.
+ */
+ if (minmax_list)
+ {
+ updt_returning =3D get_returning_string(minmax_list, is_min_list, keys);
+ select_for_recalc =3D get_select_for_recalc_string(keys);
+ }
+
/* Search for matching tuples from the view and update or delete if found=
. */
initStringInfo(&querybuf);
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
+ "%s" /* RETURNING clause for recalc infomation */
"), dlt AS (" /* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt"
- ")",
+ ") %s", /* SELECT returning which tuples need to be recalculate=
d */
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
- matviewname);
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
+ updt_returning,
+ matviewname,
+ select_for_recalc);
=20
- if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
+ if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_SELECT)
elog(ERROR, "SPI_exec failed: %s", querybuf.data);
+
+
+ /* Return tuples to be recalculated. */
+ if (minmax_list)
+ {
+ *tuptable_recalc =3D SPI_tuptable;
+ *num_recalc =3D SPI_processed;
+ }
+ else
+ {
+ *tuptable_recalc =3D NULL;
+ *num_recalc =3D 0;
+ }
}
=20
/*
@@ -2172,10 +2698,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2206,6 +2737,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2213,6 +2745,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
@@ -2287,6 +2820,349 @@ get_matching_condition_string(List *keys)
return match_cond.data;
}
=20
+/*
+ * get_returning_string
+ *
+ * Build a string for RETURNING clause of UPDATE used in apply_old_delta_w=
ith_count.
+ * This clause returns ctid and a boolean value that indicates if we need =
to
+ * recalculate min or max value, for each updated row.
+ */
+static char *
+get_returning_string(List *minmax_list, List *is_min_list, List *keys)
+{
+ StringInfoData returning;
+ char *recalc_cond;
+ ListCell *lc;
+
+ Assert(minmax_list !=3D NIL && is_min_list !=3D NIL);
+ recalc_cond =3D get_minmax_recalc_condition_string(minmax_list, is_min_li=
st);
+
+ initStringInfo(&returning);
+
+ appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", =
recalc_cond);
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", re=
sname));
+ }
+
+ return returning.data;
+}
+
+/*
+ * get_minmax_recalc_condition_string
+ *
+ * Build a predicate string for checking if any min/max aggregate
+ * value needs to be recalculated.
+ */
+static char *
+get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list)
+{
+ StringInfoData recalc_cond;
+ ListCell *lc1, *lc2;
+
+ initStringInfo(&recalc_cond);
+
+ Assert (list_length(minmax_list) =3D=3D list_length(is_min_list));
+
+ forboth (lc1, minmax_list, lc2, is_min_list)
+ {
+ char *resname =3D (char *) lfirst(lc1);
+ bool is_min =3D (bool) lfirst_int(lc2);
+ char *op_str =3D (is_min ? ">=3D" : "<=3D");
+
+ appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s",
+ quote_qualified_identifier("mv", resname),
+ op_str,
+ quote_qualified_identifier("t", resname)
+ );
+
+ if (lnext(minmax_list, lc1))
+ appendStringInfo(&recalc_cond, " OR ");
+ }
+
+ return recalc_cond.data;
+}
+
+/*
+ * get_select_for_recalc_string
+ *
+ * Build a query to return tid and keys of tuples which need
+ * recalculation. This is used as the result of the query
+ * built by apply_old_delta.
+ */
+static char *
+get_select_for_recalc_string(List *keys)
+{
+ StringInfoData qry;
+ ListCell *lc;
+
+ initStringInfo(&qry);
+
+ appendStringInfo(&qry, "SELECT tid");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ appendStringInfo(&qry, ", %s", NameStr(attr->attname));
+ }
+
+ appendStringInfo(&qry, " FROM updt WHERE recalc");
+
+ return qry.data;
+}
+
+/*
+ * recalc_and_set_values
+ *
+ * Recalculate tuples in a materialized from base tables and update these.
+ * The tuples which needs recalculation are specified by keys, and resnames
+ * of columns to be updated are specified by namelist. TIDs and key values
+ * are given by tuples in tuptable_recalc. Its first attribute must be TID
+ * and key values must be following this.
+ */
+static void
+recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples,
+ List *namelist, List *keys, Relation matviewRel)
+{
+ TupleDesc tupdesc_recalc =3D tuptable_recalc->tupdesc;
+ Oid *keyTypes =3D NULL, *types =3D NULL;
+ char *keyNulls =3D NULL, *nulls =3D NULL;
+ Datum *keyVals =3D NULL, *vals =3D NULL;
+ int num_vals =3D list_length(namelist);
+ int num_keys =3D list_length(keys);
+ uint64 i;
+ Oid matviewOid;
+ char *matviewname;
+
+ matviewOid =3D RelationGetRelid(matviewRel);
+ matviewname =3D quote_qualified_identifier(get_namespace_name(RelationGet=
Namespace(matviewRel)),
+ RelationGetRelationName(matviewRel));
+
+ /* If we have keys, initialize arrays for them. */
+ if (keys)
+ {
+ keyTypes =3D palloc(sizeof(Oid) * num_keys);
+ keyNulls =3D palloc(sizeof(char) * num_keys);
+ keyVals =3D palloc(sizeof(Datum) * num_keys);
+ /* a tuple contains keys to be recalculated and ctid to be updated*/
+ Assert(tupdesc_recalc->natts =3D=3D num_keys + 1);
+
+ /* Types of key attributes */
+ for (i =3D 0; i < num_keys; i++)
+ keyTypes[i] =3D TupleDescAttr(tupdesc_recalc, i + 1)->atttypid;
+ }
+
+ /* allocate memory for all attribute names and tid */
+ types =3D palloc(sizeof(Oid) * (num_vals + 1));
+ nulls =3D palloc(sizeof(char) * (num_vals + 1));
+ vals =3D palloc(sizeof(Datum) * (num_vals + 1));
+
+ /* For each tuple which needs recalculation */
+ for (i =3D 0; i < num_tuples; i++)
+ {
+ int j;
+ bool isnull;
+ SPIPlanPtr plan;
+ SPITupleTable *tuptable_newvals;
+ TupleDesc tupdesc_newvals;
+
+ /* Set group key values as parameters if needed. */
+ if (keys)
+ {
+ for (j =3D 0; j < num_keys; j++)
+ {
+ keyVals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc,=
j + 2, &isnull);
+ if (isnull)
+ keyNulls[j] =3D 'n';
+ else
+ keyNulls[j] =3D ' ';
+ }
+ }
+
+ /*
+ * Get recalculated values from base tables. The result must be
+ * only one tuple thich contains the new values for specified keys.
+ */
+ plan =3D get_plan_for_recalc(matviewOid, namelist, keys, keyTypes);
+ if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) !=3D SPI_OK_SELE=
CT)
+ elog(ERROR, "SPI_execute_plan");
+ if (SPI_processed !=3D 1)
+ elog(ERROR, "SPI_execute_plan returned zero or more than one rows");
+
+ tuptable_newvals =3D SPI_tuptable;
+ tupdesc_newvals =3D tuptable_newvals->tupdesc;
+
+ Assert(tupdesc_newvals->natts =3D=3D num_vals);
+
+ /* Set the new values as parameters */
+ for (j =3D 0; j < tupdesc_newvals->natts; j++)
+ {
+ if (i =3D=3D 0)
+ types[j] =3D TupleDescAttr(tupdesc_newvals, j)->atttypid;
+
+ vals[j] =3D SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j=
+ 1, &isnull);
+ if (isnull)
+ nulls[j] =3D 'n';
+ else
+ nulls[j] =3D ' ';
+ }
+ /* Set TID of the view tuple to be updated as a parameter */
+ types[j] =3D TIDOID;
+ vals[j] =3D SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &=
isnull);
+ nulls[j] =3D ' ';
+
+ /* Update the view tuple to the new values */
+ plan =3D get_plan_for_set_values(matviewOid, matviewname, namelist, type=
s);
+ if (SPI_execute_plan(plan, vals, nulls, false, 0) !=3D SPI_OK_UPDATE)
+ elog(ERROR, "SPI_execute_plan");
+ }
+}
+
+
+/*
+ * get_plan_for_recalc
+ *
+ * Create or fetch a plan for recalculating value in the view's target list
+ * from base tables using the definition query of materialized view specif=
ied
+ * by matviewOid. namelist is a list of resnames of values to be recalcula=
ted.
+ *
+ * keys is a list of keys to identify tuples to be recalculated if this is=
not
+ * empty. KeyTypes is an array of types of keys.
+ */
+static SPIPlanPtr
+get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTy=
pes)
+{
+ MV_QueryKey hash_key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the recalculation */
+ mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC);
+ if ((plan =3D mv_FetchPreparedPlan(&hash_key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ char *viewdef;
+
+ /* get view definition of matview */
+ viewdef =3D text_to_cstring((text *) DatumGetPointer(
+ DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid))));
+ /* get rid of trailing semi-colon */
+ viewdef[strlen(viewdef)-1] =3D '\0';
+
+ /*
+ * Build a query string for recalculating values. This is like
+ *
+ * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv
+ * WHERE (key1, key2, ...) =3D ($1, $2, ...);
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "SELECT ");
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, " FROM (%s) mv", viewdef);
+
+ if (keys)
+ {
+ int i =3D 1;
+ char paramname[16];
+
+ appendStringInfo(&str, " WHERE (");
+ foreach (lc, keys)
+ {
+ Form_pg_attribute attr =3D (Form_pg_attribute) lfirst(lc);
+ char *resname =3D NameStr(attr->attname);
+ Oid typid =3D attr->atttypid;
+
+ sprintf(paramname, "$%d", i);
+ appendStringInfo(&str, "(");
+ generate_equal(&str, typid, resname, paramname);
+ appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))",
+ resname, paramname);
+
+ if (lnext(keys, lc))
+ appendStringInfoString(&str, " AND ");
+ i++;
+ }
+ appendStringInfo(&str, ")");
+ }
+ else
+ keyTypes =3D NULL;
+
+ plan =3D SPI_prepare(str.data, list_length(keys), keyTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&hash_key, plan);
+ }
+
+ return plan;
+}
+
+/*
+ * get_plan_for_set_values
+ *
+ * Create or fetch a plan for applying new values calculated by
+ * get_plan_for_recalc to a materialized view specified by matviewOid.
+ * matviewname is the name of the view. namelist is a list of resnames
+ * of attributes to be updated, and valTypes is an array of types of the
+ * values.
+ */
+static SPIPlanPtr
+get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist,
+ Oid *valTypes)
+{
+ MV_QueryKey key;
+ SPIPlanPtr plan;
+
+ /* Fetch or prepare a saved plan for the real check */
+ mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE);
+ if ((plan =3D mv_FetchPreparedPlan(&key)) =3D=3D NULL)
+ {
+ ListCell *lc;
+ StringInfoData str;
+ int i;
+
+ /*
+ * Build a query string for applying min/max values. This is like
+ *
+ * UPDATE matviewname AS mv
+ * SET (x1, x2, x3, x4) =3D ($1, $2, $3, $4)
+ * WHERE ctid =3D $5;
+ */
+
+ initStringInfo(&str);
+ appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname);
+ foreach (lc, namelist)
+ {
+ appendStringInfo(&str, "%s", (char *) lfirst(lc));
+ if (lnext(namelist, lc))
+ appendStringInfoString(&str, ", ");
+ }
+ appendStringInfo(&str, ") =3D ROW(");
+
+ for (i =3D 1; i <=3D list_length(namelist); i++)
+ appendStringInfo(&str, "%s$%d", (i=3D=3D1 ? "" : ", "), i);
+
+ appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=3D) $%d", i);
+
+ plan =3D SPI_prepare(str.data, list_length(namelist) + 1, valTypes);
+ if (plan =3D=3D NULL)
+ elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SP=
I_result), str.data);
+
+ SPI_keepplan(plan);
+ mv_HashPreparedPlan(&key, plan);
+ }
+
+ return plan;
+}
+
/*
* generate_equals
*
@@ -2320,6 +3196,13 @@ mv_InitHashTables(void)
{
HASHCTL ctl;
=20
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize =3D sizeof(MV_QueryKey);
+ ctl.entrysize =3D sizeof(MV_QueryHashEntry);
+ mv_query_cache =3D hash_create("MV query cache",
+ MV_INIT_QUERYHASHSIZE,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
memset(&ctl, 0, sizeof(ctl));
ctl.keysize =3D sizeof(Oid);
ctl.entrysize =3D sizeof(MV_TriggerHashEntry);
@@ -2328,6 +3211,99 @@ mv_InitHashTables(void)
&ctl, HASH_ELEM | HASH_BLOBS);
}
=20
+/*
+ * mv_FetchPreparedPlan
+ */
+static SPIPlanPtr
+mv_FetchPreparedPlan(MV_QueryKey *key)
+{
+ MV_QueryHashEntry *entry;
+ SPIPlanPtr plan;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Lookup for the key
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_FIND, NULL);
+ if (entry =3D=3D NULL)
+ return NULL;
+
+ /*
+ * Check whether the plan is still valid. If it isn't, we don't want to
+ * simply rely on plancache.c to regenerate it; rather we should start
+ * from scratch and rebuild the query text too. This is to cover cases
+ * such as table/column renames. We depend on the plancache machinery to
+ * detect possible invalidations, though.
+ *
+ * CAUTION: this check is only trustworthy if the caller has already
+ * locked both materialized views and base tables.
+ */
+ plan =3D entry->plan;
+ if (plan && SPI_plan_is_valid(plan))
+ return plan;
+
+ /*
+ * Otherwise we might as well flush the cached plan now, to free a little
+ * memory space before we make a new one.
+ */
+ entry->plan =3D NULL;
+ if (plan)
+ SPI_freeplan(plan);
+
+ return NULL;
+}
+
+/*
+ * mv_HashPreparedPlan
+ *
+ * Add another plan to our private SPI query plan hashtable.
+ */
+static void
+mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan)
+{
+ MV_QueryHashEntry *entry;
+ bool found;
+
+ /*
+ * On the first call initialize the hashtable
+ */
+ if (!mv_query_cache)
+ mv_InitHashTables();
+
+ /*
+ * Add the new plan. We might be overwriting an entry previously found
+ * invalid by mv_FetchPreparedPlan.
+ */
+ entry =3D (MV_QueryHashEntry *) hash_search(mv_query_cache,
+ (void *) key,
+ HASH_ENTER, &found);
+ Assert(!found || entry->plan =3D=3D NULL);
+ entry->plan =3D plan;
+}
+
+/*
+ * mv_BuildQueryKey
+ *
+ * Construct a hashtable key for a prepared SPI plan for IVM.
+ */
+static void
+mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type)
+{
+ /*
+ * We assume struct MV_QueryKey contains no padding bytes, else we'd need
+ * to use memset to clear them.
+ */
+ key->matview_id =3D matview_id;
+ key->query_type =3D query_type;
+}
+
/*
* AtAbort_IVM
*
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bcea9782d3..e36302845f 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid, bool is_cr
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.17.1
--Multipart=_Fri__4_Feb_2022_01_48_06_+0900_N8BNZpfOR27sWrgY
Content-Type: text/x-diff;
name="v25-0007-Add-Incremental-View-Maintenance-support.patch"
Content-Disposition: attachment;
filename="v25-0007-Add-Incremental-View-Maintenance-support.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v38 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 275 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 672 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..35e124694ab 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,19 +448,61 @@ rewriteQueryForIMMV(Query *query, List *colNames)
ParseState *pstate =3D make_parsestate(NULL);
FuncCall *fn;
=20
+ /*
+ * Check the length of column name list not to override names of
+ * additional columns
+ */
+ if (list_length(colNames) > list_length(query->targetList))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("too many column names were specified")));
+
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +519,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1087,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1122,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1193,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1243,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1254,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1271,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1308,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1405,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1485,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ
Content-Type: text/x-diff;
name="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v30 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cce44278fa..d93eec3eec 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index dbcbc79fff..3c523991ed 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1453,11 +1475,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1497,8 +1552,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1549,7 +1604,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1945,17 +2000,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2028,6 +2100,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2041,6 +2115,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2063,6 +2140,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2079,13 +2165,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2116,7 +2250,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2142,7 +2277,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2157,6 +2292,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2164,13 +2543,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2180,22 +2566,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2259,10 +2649,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2293,6 +2688,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2300,6 +2696,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
name="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v30 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cce44278fa..d93eec3eec 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index dbcbc79fff..3c523991ed 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1453,11 +1475,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1497,8 +1552,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1549,7 +1604,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1945,17 +2000,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2028,6 +2100,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2041,6 +2115,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2063,6 +2140,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2079,13 +2165,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2116,7 +2250,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2142,7 +2277,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2157,6 +2292,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2164,13 +2543,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2180,22 +2566,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2259,10 +2649,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2293,6 +2688,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2300,6 +2696,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
name="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v30 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cce44278fa..d93eec3eec 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index dbcbc79fff..3c523991ed 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1453,11 +1475,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1497,8 +1552,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1549,7 +1604,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1945,17 +2000,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2028,6 +2100,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2041,6 +2115,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2063,6 +2140,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2079,13 +2165,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2116,7 +2250,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2142,7 +2277,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2157,6 +2292,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2164,13 +2543,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2180,22 +2566,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2259,10 +2649,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2293,6 +2688,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2300,6 +2696,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
name="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v31 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 265 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 662 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 8f2bd5203e..aa8440b4e1 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,13 +51,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -71,6 +77,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -85,8 +96,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -417,6 +429,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -430,16 +443,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -456,6 +502,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -939,11 +1070,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -972,6 +1105,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1024,6 +1161,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1072,7 +1211,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1083,8 +1222,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1096,14 +1239,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1111,6 +1276,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1168,7 +1373,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1226,7 +1453,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index f2e8aa02a3..97406b28c9 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -27,6 +27,7 @@
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -107,6 +109,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -138,7 +147,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -149,14 +158,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1452,11 +1474,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1496,8 +1551,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1548,7 +1603,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1944,17 +1999,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2027,6 +2099,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2040,6 +2114,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2062,6 +2139,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2078,13 +2164,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2115,7 +2249,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2141,7 +2276,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2156,6 +2291,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2163,13 +2542,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2179,22 +2565,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2258,10 +2648,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2292,6 +2687,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2299,6 +2695,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Fri__29_Mar_2024_23_47_00_+0900_KGpmmDOIs1266Ib1
Content-Type: text/x-diff;
name="v31-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v31-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v32 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 265 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 662 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 299c5a133c..ecec93ec1c 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,13 +51,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -71,6 +77,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -85,8 +96,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -417,6 +429,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -430,16 +443,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -456,6 +502,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -941,11 +1072,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -974,6 +1107,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1026,6 +1163,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1074,7 +1213,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1085,8 +1224,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1098,14 +1241,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1113,6 +1278,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1170,7 +1375,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1228,7 +1455,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 0064e10966..b7f6c3831b 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -27,6 +27,7 @@
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -107,6 +109,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -138,7 +147,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -149,14 +158,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1454,11 +1476,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1498,8 +1553,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1550,7 +1605,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1946,17 +2001,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2029,6 +2101,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2042,6 +2116,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2064,6 +2141,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2080,13 +2166,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2117,7 +2251,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2143,7 +2278,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2158,6 +2293,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2165,13 +2544,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2181,22 +2567,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2260,10 +2650,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2294,6 +2689,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2301,6 +2697,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Sun__31_Mar_2024_22_59_31_+0900_msknEviJj08_wgqO
Content-Type: text/x-diff;
name="v32-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v32-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v28 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 076f35ee6b..c8aa558f2e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index 6d8382180a..aa6bf2694a 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1432,11 +1454,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1476,8 +1531,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1528,7 +1583,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1924,17 +1979,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2007,6 +2079,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2020,6 +2094,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2042,6 +2119,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2058,13 +2144,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2095,7 +2229,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2121,7 +2256,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2136,6 +2271,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2143,13 +2522,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2159,22 +2545,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2238,10 +2628,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2272,6 +2667,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2279,6 +2675,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 76a7873ebf..599bae3b5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Thu__1_Jun_2023_23_59_09_+0900_/G5+8nG46.f1T42K
Content-Type: text/x-diff;
name="v28-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v28-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v29 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 076f35ee6b..c8aa558f2e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index aa518f20ef..ee41f0007d 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1431,11 +1453,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1475,8 +1530,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1527,7 +1582,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1923,17 +1978,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2006,6 +2078,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2019,6 +2093,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2041,6 +2118,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2057,13 +2143,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2094,7 +2228,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2120,7 +2255,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2135,6 +2270,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2142,13 +2521,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2158,22 +2544,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2237,10 +2627,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2271,6 +2666,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2278,6 +2674,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 76a7873ebf..599bae3b5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7
Content-Type: text/x-diff;
name="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v29 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 076f35ee6b..c8aa558f2e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index aa518f20ef..ee41f0007d 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1431,11 +1453,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1475,8 +1530,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1527,7 +1582,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1923,17 +1978,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2006,6 +2078,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2019,6 +2093,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2041,6 +2118,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2057,13 +2143,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2094,7 +2228,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2120,7 +2255,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2135,6 +2270,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2142,13 +2521,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2158,22 +2544,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2237,10 +2627,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2271,6 +2666,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2278,6 +2674,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 76a7873ebf..599bae3b5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__28_Aug_2023_16_05_30_+0900_b1OvQD_3A3ZMTGvj
Content-Type: text/x-diff;
name="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v30 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cce44278fa..d93eec3eec 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index dbcbc79fff..3c523991ed 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1453,11 +1475,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1497,8 +1552,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1549,7 +1604,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1945,17 +2000,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2028,6 +2100,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2041,6 +2115,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2063,6 +2140,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2079,13 +2165,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2116,7 +2250,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2142,7 +2277,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2157,6 +2292,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2164,13 +2543,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2180,22 +2566,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2259,10 +2649,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2293,6 +2688,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2300,6 +2696,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 6b47e66bfd..af3a5b4b27 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
name="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v30-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v37 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 266 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 663 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..45a30309951 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -437,16 +451,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +510,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1078,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1113,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1184,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1234,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1245,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1262,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1299,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1396,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1476,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd
Content-Type: text/x-diff;
name="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v38 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 275 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 672 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..35e124694ab 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,19 +448,61 @@ rewriteQueryForIMMV(Query *query, List *colNames)
ParseState *pstate =3D make_parsestate(NULL);
FuncCall *fn;
=20
+ /*
+ * Check the length of column name list not to override names of
+ * additional columns
+ */
+ if (list_length(colNames) > list_length(query->targetList))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("too many column names were specified")));
+
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +519,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1087,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1122,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1193,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1243,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1254,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1271,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1308,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1405,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1485,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ
Content-Type: text/x-diff;
name="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v29 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 076f35ee6b..c8aa558f2e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index aa518f20ef..ee41f0007d 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1431,11 +1453,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1475,8 +1530,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1527,7 +1582,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1923,17 +1978,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2006,6 +2078,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2019,6 +2093,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2041,6 +2118,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2057,13 +2143,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2094,7 +2228,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2120,7 +2255,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2135,6 +2270,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2142,13 +2521,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2158,22 +2544,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2237,10 +2627,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2271,6 +2666,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2278,6 +2674,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 76a7873ebf..599bae3b5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7
Content-Type: text/x-diff;
name="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v29 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 264 +++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 661 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index 076f35ee6b..c8aa558f2e 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -54,14 +54,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +80,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +99,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *q=
ry, Node *node, Oid mat
Relids *relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -421,6 +432,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,16 +446,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -460,6 +505,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -943,11 +1073,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -976,6 +1108,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1028,6 +1164,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
}
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1076,7 +1214,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1087,8 +1225,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1100,14 +1242,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1115,6 +1279,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1172,7 +1376,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1230,7 +1456,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index aa518f20ef..ee41f0007d 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -30,6 +30,7 @@
#include "catalog/pg_opclass.h"
#include "catalog/pg_operator.h"
#include "commands/cluster.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -39,6 +40,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -111,6 +113,13 @@ static HTAB *mv_trigger_info =3D NULL;
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -142,7 +151,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *replace_rte_with_delta(RangeTblEntry *rte, MV_Trigge=
rTable *table, bool is_new,
QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate);
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -153,14 +162,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1431,11 +1453,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time it
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "");
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false);
@@ -1475,8 +1530,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1527,7 +1582,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -1923,17 +1978,34 @@ replace_rte_with_delta(RangeTblEntry *rte, MV_Trigg=
erTable *table, bool is_new,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2006,6 +2078,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2019,6 +2093,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2041,6 +2118,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2057,13 +2143,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if these
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2094,7 +2228,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2120,7 +2255,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2135,6 +2270,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col);
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2142,13 +2521,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given by
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2158,22 +2544,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2237,10 +2627,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a keys
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2271,6 +2666,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2278,6 +2674,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index 76a7873ebf..599bae3b5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.25.1
--Multipart=_Mon__28_Aug_2023_16_05_30_+0900_b1OvQD_3A3ZMTGvj
Content-Type: text/x-diff;
name="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Disposition: attachment;
filename="v29-0009-Add-support-for-min-max-aggregates-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v37 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 266 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 663 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..45a30309951 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -437,16 +451,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +510,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1078,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1113,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1184,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1234,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1245,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1262,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1299,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1396,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1476,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd
Content-Type: text/x-diff;
name="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v37 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 266 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 663 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..45a30309951 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -437,16 +451,49 @@ rewriteQueryForIMMV(Query *query, List *colNames)
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +510,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1078,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1113,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1184,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1234,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1245,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1262,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1299,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1396,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1476,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd
Content-Type: text/x-diff;
name="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v37-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v38 08/11] Add aggregates support in IVM
@ 2023-05-31 11:46 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Yugo Nagata @ 2023-05-31 11:46 UTC (permalink / raw)
count, sum, adn avg are supported.
As a restriction, expressions specified in GROUP BY must appear in
the target list because tuples to be updated in IMMV are identified
by using this group key. However, in the case of aggregates without
GROUP BY, there is only one tuple in the view, so keys are not uses
to identify tuples.
When creating a IMMV, in addition to __ivm_count column, some hidden
columns for each aggregate are added to the target list. For example,
names of these hidden columns are ivm_count_avg and ivm_sum_avg for
the average function, and so on.
When a base table is modified, the aggregated values and related
hidden columns are also updated as well as __ivm_count__. The
way of update depends the kind of aggregate function.=E3=80=80Specifically,
sum and count are updated by simply adding or subtracting delta value
calculated from delta tables. avg is updated by using values of sum
and count stored in views as hidden columns and deltas calculated
from delta tables.
About aggregate functions except "count()" (sum and avg), NULLs in input
values are ignored, and the result of aggegate should be NULL when no
rows are selected. To support this specification, the numbers of non-NULL
input values are counted and stored in hidden columns. In the case of
count(), count(x) returns zero when no rows are selected, but count(*)
doesn't ignore NULL input.
---
src/backend/commands/createas.c | 275 ++++++++++++++++++--
src/backend/commands/matview.c | 433 ++++++++++++++++++++++++++++++--
src/include/commands/createas.h | 1 +
3 files changed, 672 insertions(+), 37 deletions(-)
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createa=
s.c
index cd8db0059f9..35e124694ab 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -56,12 +56,19 @@
#include "parser/parsetree.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
+#include "parser/parse_type.h"
#include "rewrite/rewriteHandler.h"
+#include "rewrite/rewriteManip.h"
+#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
+#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
+#include "utils/regproc.h"
+#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
+#include "utils/syscache.h"
=20
typedef struct
{
@@ -75,6 +82,11 @@ typedef struct
BulkInsertState bistate; /* bulk insert state */
} DR_intorel;
=20
+typedef struct
+{
+ bool has_agg;
+} check_ivm_restriction_context;
+
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into=
);
static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -89,8 +101,9 @@ static void CreateIvmTriggersOnBaseTablesRecurse(Query *=
qry, Node *node, Oid mat
List **relids, bool ex_lock);
static void CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type, int16 ti=
ming, bool ex_lock);
static void check_ivm_restriction(Node *node);
-static bool check_ivm_restriction_walker(Node *node, void *context);
+static bool check_ivm_restriction_walker(Node *node, check_ivm_restriction=
_context *context);
static Bitmapset *get_primary_key_attnos_from_query(Query *query, List **c=
onstraintList);
+static bool check_aggregate_supports_ivm(Oid aggfnoid);
=20
/*
* create_ctas_internal
@@ -424,6 +437,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt=
*stmt,
* rewriteQueryForIMMV -- rewrite view definition query for IMMV
*
* count(*) is added for counting distinct tuples in views.
+ * Also, additional hidden columns are added for aggregate values.
*/
Query *
rewriteQueryForIMMV(Query *query, List *colNames)
@@ -434,19 +448,61 @@ rewriteQueryForIMMV(Query *query, List *colNames)
ParseState *pstate =3D make_parsestate(NULL);
FuncCall *fn;
=20
+ /*
+ * Check the length of column name list not to override names of
+ * additional columns
+ */
+ if (list_length(colNames) > list_length(query->targetList))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("too many column names were specified")));
+
rewritten =3D copyObject(query);
pstate->p_expr_kind =3D EXPR_KIND_SELECT_TARGET;
=20
- /*
- * Convert DISTINCT to GROUP BY and add count(*) for counting distinct
- * tuples in views.
- */
- if (rewritten->distinctClause)
+ /* group keys must be in targetlist */
+ if (rewritten->groupClause)
{
- TargetEntry *tle;
+ ListCell *lc;
+ foreach(lc, rewritten->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, rewritten->targetList=
);
=20
+ if (tle->resjunk)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY expression not appearing in select list is not sup=
ported on incrementally maintainable materialized view")));
+ }
+ }
+ /* Convert DISTINCT to GROUP BY. count(*) will be added afterward. */
+ else if (!rewritten->hasAggs && rewritten->distinctClause)
rewritten->groupClause =3D transformDistinctClause(NULL, &rewritten->tar=
getList, rewritten->sortClause, false);
=20
+ /* Add additional columns for aggregate values */
+ if (rewritten->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(rewritten->targetList) + 1;
+
+ foreach(lc, rewritten->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+ char *resname =3D (colNames =3D=3D NIL || foreach_current_index(lc) >=
=3D list_length(colNames) ?
+ tle->resname : strVal(list_nth(colNames, tle->resno - 1)));
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *) tle->expr, resname, &next_resno, &=
aggs);
+ }
+ rewritten->targetList =3D list_concat(rewritten->targetList, aggs);
+ }
+
+ /* Add count(*) for counting distinct tuples in views */
+ if (rewritten->distinctClause || rewritten->hasAggs)
+ {
+ TargetEntry *tle;
+
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
fn->agg_star =3D true;
=20
@@ -463,6 +519,91 @@ rewriteQueryForIMMV(Query *query, List *colNames)
return rewritten;
}
=20
+/*
+ * makeIvmAggColumn -- make additional aggregate columns for IVM
+ *
+ * For an aggregate column specified by aggref, additional aggregate colum=
ns
+ * are added, which are used to calculate the new aggregate value in IMMV.
+ * An additional aggregate columns has a name based on resname
+ * (ex. ivm_count_resname), and resno specified by next_resno. The created
+ * columns are returned to aggs, and the resno for the next column is also
+ * returned to next_resno.
+ *
+ * Currently, an additional count() is created for aggref other than count=
.
+ * In addition, sum() is created for avg aggregate column.
+ */
+void
+makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *resname, AttrNu=
mber *next_resno, List **aggs)
+{
+ TargetEntry *tle_count;
+ Node *node;
+ FuncCall *fn;
+ Const *dmy_arg =3D makeConst(INT4OID,
+ -1,
+ InvalidOid,
+ sizeof(int32),
+ Int32GetDatum(1),
+ false,
+ true); /* pass by value */
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * For aggregate functions except count, add count() func with the same a=
rg parameters.
+ * This count result is used for determining if the aggregate value shoul=
d be NULL or not.
+ * Also, add sum() func for avg because we need to calculate an average v=
alue as sum/count.
+ *
+ * XXX: If there are same expressions explicitly in the target list, we c=
an use this instead
+ * of adding new duplicated one.
+ */
+ if (strcmp(aggname, "count") !=3D 0)
+ {
+ fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, =
-1);
+
+ /* Make a Func with a dummy arg, and then override this by the original =
agg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, list_make1(dmy_arg), NU=
LL, fn, false, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_count",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+ if (strcmp(aggname, "avg") =3D=3D 0)
+ {
+ List *dmy_args =3D NIL;
+ ListCell *lc;
+ foreach(lc, aggref->aggargtypes)
+ {
+ Oid typeid =3D lfirst_oid(lc);
+ Type type =3D typeidType(typeid);
+
+ Const *con =3D makeConst(typeid,
+ -1,
+ typeTypeCollation(type),
+ typeLen(type),
+ (Datum) 0,
+ true,
+ typeByVal(type));
+ dmy_args =3D lappend(dmy_args, con);
+ ReleaseSysCache(type);
+ }
+ fn =3D makeFuncCall(SystemFuncName("sum"), NIL, COERCE_EXPLICIT_CALL, -1=
);
+
+ /* Make a Func with dummy args, and then override this by the original a=
gg's args. */
+ node =3D ParseFuncOrColumn(pstate, fn->funcname, dmy_args, NULL, fn, fal=
se, -1);
+ ((Aggref *)node)->args =3D aggref->args;
+
+ tle_count =3D makeTargetEntry((Expr *) node,
+ *next_resno,
+ pstrdup(makeObjectName("__ivm_sum",resname, "_")),
+ false);
+ *aggs =3D lappend(*aggs, tle_count);
+ (*next_resno)++;
+ }
+}
+
/*
* GetIntoRelEFlags --- compute executor flags needed for CREATE TABLE AS
*
@@ -946,11 +1087,13 @@ CreateIvmTrigger(Oid relOid, Oid viewOid, int16 type=
, int16 timing, bool ex_lock
static void
check_ivm_restriction(Node *node)
{
- check_ivm_restriction_walker(node, NULL);
+ check_ivm_restriction_context context =3D {false};
+
+ check_ivm_restriction_walker(node, &context);
}
=20
static bool
-check_ivm_restriction_walker(Node *node, void *context)
+check_ivm_restriction_walker(Node *node, check_ivm_restriction_context *co=
ntext)
{
if (node =3D=3D NULL)
return false;
@@ -979,6 +1122,10 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("CTE is not supported on incrementally maintainable materia=
lized view")));
+ if (qry->groupClause !=3D NIL && !qry->hasAggs)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("GROUP BY clause without aggregate is not supported on incr=
ementally maintainable materialized view")));
if (qry->havingQual !=3D NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1046,6 +1193,8 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
format_type_be(atttype), "btree")));
}
=20
+ context->has_agg |=3D qry->hasAggs;
+
/* restrictions for rtable */
foreach(lc, qry->rtable)
{
@@ -1094,7 +1243,7 @@ check_ivm_restriction_walker(Node *node, void *contex=
t)
=20
}
=20
- query_tree_walker(qry, check_ivm_restriction_walker, NULL, QTW_IGNORE_=
RANGE_TABLE);
+ query_tree_walker(qry, check_ivm_restriction_walker, (void *) context,=
QTW_IGNORE_RANGE_TABLE);
=20
break;
}
@@ -1105,8 +1254,12 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("column name %s is not supported on incrementally maintain=
able materialized view", tle->resname)));
+ if (context->has_agg && !IsA(tle->expr, Aggref) && contain_aggs_of_lev=
el((Node *) tle->expr, 0))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("expression containing an aggregate in it is not supported =
on incrementally maintainable materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
break;
}
case T_JoinExpr:
@@ -1118,14 +1271,36 @@ check_ivm_restriction_walker(Node *node, void *cont=
ext)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("OUTER JOIN is not supported on incrementally maintainable=
materialized view")));
=20
- expression_tree_walker(node, check_ivm_restriction_walker, NULL);
+ expression_tree_walker(node, check_ivm_restriction_walker, (void *) co=
ntext);
+ break;
}
- break;
case T_Aggref:
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("aggregate function is not supported on incrementally maintai=
nable materialized view")));
- break;
+ {
+ /* Check if this supports IVM */
+ Aggref *aggref =3D (Aggref *) node;
+ const char *aggname =3D format_procedure(aggref->aggfnoid);
+
+ if (aggref->aggfilter !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with FILTER clause is not supported on =
incrementally maintainable materialized view")));
+
+ if (aggref->aggdistinct !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with DISTINCT arguments is not supporte=
d on incrementally maintainable materialized view")));
+
+ if (aggref->aggorder !=3D NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function with ORDER clause is not supported on i=
ncrementally maintainable materialized view")));
+
+ if (!check_aggregate_supports_ivm(aggref->aggfnoid))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("aggregate function %s is not supported on incrementally ma=
intainable materialized view", aggname)));
+ break;
+ }
default:
expression_tree_walker(node, check_ivm_restriction_walker, (void *) con=
text);
break;
@@ -1133,6 +1308,46 @@ check_ivm_restriction_walker(Node *node, void *conte=
xt)
return false;
}
=20
+/*
+ * check_aggregate_supports_ivm
+ *
+ * Check if the given aggregate function is supporting IVM
+ */
+static bool
+check_aggregate_supports_ivm(Oid aggfnoid)
+{
+ switch (aggfnoid)
+ {
+ /* count */
+ case F_COUNT_ANY:
+ case F_COUNT_:
+
+ /* sum */
+ case F_SUM_INT8:
+ case F_SUM_INT4:
+ case F_SUM_INT2:
+ case F_SUM_FLOAT4:
+ case F_SUM_FLOAT8:
+ case F_SUM_MONEY:
+ case F_SUM_INTERVAL:
+ case F_SUM_NUMERIC:
+
+ /* avg */
+ case F_AVG_INT8:
+ case F_AVG_INT4:
+ case F_AVG_INT2:
+ case F_AVG_NUMERIC:
+ case F_AVG_FLOAT4:
+ case F_AVG_FLOAT8:
+ case F_AVG_INTERVAL:
+
+ return true;
+
+ default:
+ return false;
+ }
+}
+
/*
* CreateIndexOnIMMV
*
@@ -1190,7 +1405,29 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
index->concurrent =3D false;
index->if_not_exists =3D false;
=20
- if (query->distinctClause)
+ if (query->groupClause)
+ {
+ /* create unique constraint on GROUP BY expression columns */
+ foreach(lc, query->groupClause)
+ {
+ SortGroupClause *scl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(scl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+ IndexElem *iparam;
+
+ iparam =3D makeNode(IndexElem);
+ iparam->name =3D pstrdup(NameStr(attr->attname));
+ iparam->expr =3D NULL;
+ iparam->indexcolname =3D NULL;
+ iparam->collation =3D NIL;
+ iparam->opclass =3D NIL;
+ iparam->opclassopts =3D NIL;
+ iparam->ordering =3D SORTBY_DEFAULT;
+ iparam->nulls_ordering =3D SORTBY_NULLS_DEFAULT;
+ index->indexParams =3D lappend(index->indexParams, iparam);
+ }
+ }
+ else if (query->distinctClause)
{
/* create unique constraint on all columns */
foreach(lc, query->targetList)
@@ -1248,7 +1485,7 @@ CreateIndexOnIMMV(Query *query, Relation matviewRel)
(errmsg("could not create an index on materialized view \"%s\" automa=
tically",
RelationGetRelationName(matviewRel)),
errdetail("This target list does not have all the primary key column=
s, "
- "or this view does not contain DISTINCT clause."),
+ "or this view does not contain GROUP BY or DISTINCT clause."),
errhint("Create an index on the materialized view for efficient incr=
emental maintenance.")));
return;
}
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.=
c
index a2746ca9265..710224aa994 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -26,6 +26,7 @@
#include "catalog/pg_depend.h"
#include "catalog/pg_trigger.h"
#include "catalog/pg_opclass.h"
+#include "commands/defrem.h"
#include "commands/matview.h"
#include "commands/repack.h"
#include "commands/tablecmds.h"
@@ -36,6 +37,7 @@
#include "executor/tstoreReceiver.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "optimizer/optimizer.h"
#include "parser/analyze.h"
#include "parser/parse_clause.h"
#include "parser/parse_func.h"
@@ -152,6 +154,13 @@ IvmShmemRequest(void *arg)
=20
static bool in_delta_calculation =3D false;
=20
+/* kind of IVM operation for the view */
+typedef enum
+{
+ IVM_ADD,
+ IVM_SUB
+} IvmOp;
+
/* ENR name for materialized view delta */
#define NEW_DELTA_ENRNAME "new_delta"
#define OLD_DELTA_ENRNAME "old_delta"
@@ -183,7 +192,7 @@ static RangeTblEntry *get_prestate_rte(RangeTblEntry *r=
te, MV_TriggerTable *tabl
QueryEnvironment *queryEnv, Oid matviewid);
static RangeTblEntry *makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable *=
table,
bool is_new, QueryEnvironment *queryEnv);
-static Query *rewrite_query_for_counting(Query *query, ParseState *pstate)=
;
+static Query *rewrite_query_for_counting_and_aggregates(Query *query, Pars=
eState *pstate);
=20
static void calc_delta(MV_TriggerTable *table, int rte_index, Query *query=
,
DestReceiver *dest_old, DestReceiver *dest_new,
@@ -194,14 +203,27 @@ static Query *rewrite_query_for_postupdate_state(Quer=
y *query, MV_TriggerTable *
static void apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, =
Tuplestorestate *new_tuplestores,
TupleDesc tupdesc_old, TupleDesc tupdesc_new,
Query *query, bool use_count, char *count_colname);
+static void append_set_clause_for_count(const char *resname, StringInfo bu=
f_old,
+ StringInfo buf_new,StringInfo aggs_list);
+static void append_set_clause_for_sum(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list);
+static void append_set_clause_for_avg(const char *resname, StringInfo buf_=
old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype);
+static char *get_operation_string(IvmOp op, const char *col, const char *a=
rg1, const char *arg2,
+ const char* count_col, const char *castType);
+static char *get_null_condition_string(IvmOp op, const char *arg1, const c=
har *arg2,
+ const char* count_col);
static void apply_old_delta(const char *matviewname, const char *deltaname=
_old,
List *keys);
static void apply_old_delta_with_count(const char *matviewname, const char=
*deltaname_old,
- List *keys, const char *count_colname);
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname);
static void apply_new_delta(const char *matviewname, const char *deltaname=
_new,
StringInfo target_list);
static void apply_new_delta_with_count(const char *matviewname, const char=
* deltaname_new,
- List *keys, StringInfo target_list, const char* count_colname);
+ List *keys, StringInfo target_list, StringInfo aggs_set,
+ const char* count_colname);
static char *get_matching_condition_string(List *keys);
static void generate_equal(StringInfo querybuf, Oid opttype,
const char *leftop, const char *rightop);
@@ -1607,11 +1629,44 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
* When a base table is truncated, the view content will be empty if the
* view definition query does not contain an aggregate without a GROUP cl=
ause.
* Therefore, such views can be truncated.
+ *
+ * Aggregate views without a GROUP clause always have one row. Therefore,
+ * if a base table is truncated, the view will not be empty and will cont=
ain
+ * a row with NULL value (or 0 for count()). So, in this case, we refresh=
the
+ * view instead of truncating it.
*/
if (TRIGGER_FIRED_BY_TRUNCATE(trigdata->tg_event))
{
- ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
- NIL, DROP_RESTRICT, false, false);
+ if (!(query->hasAggs && query->groupClause =3D=3D NIL))
+ ExecuteTruncateGuts(list_make1(matviewRel), list_make1_oid(matviewOid),
+ NIL, DROP_RESTRICT, false, false);
+ else
+ {
+ Oid OIDNewHeap;
+ DestReceiver *dest;
+ uint64 processed =3D 0;
+ Query *dataQuery =3D rewriteQueryForIMMV(query, NIL);
+ char relpersistence =3D matviewRel->rd_rel->relpersistence;
+
+ /*
+ * Create the transient table that will receive the regenerated data. L=
ock
+ * it against access by any other process until commit (by which time i=
t
+ * will be gone).
+ */
+ OIDNewHeap =3D make_new_heap(matviewOid, matviewRel->rd_rel->reltablesp=
ace,
+ matviewRel->rd_rel->relam,
+ relpersistence, ExclusiveLock);
+ LockRelationOid(OIDNewHeap, AccessExclusiveLock);
+ dest =3D CreateTransientRelDestReceiver(OIDNewHeap);
+
+ /* Generate the data */
+ processed =3D refresh_matview_datafill(dest, dataQuery, NULL, NULL, "",=
false);
+ refresh_by_heap_swap(matviewOid, OIDNewHeap, relpersistence);
+
+ /* Inform cumulative stats system about our activity */
+ pgstat_count_truncate(matviewRel);
+ pgstat_count_heap_insert(matviewRel, processed);
+ }
=20
/* Clean up hash entry and delete tuplestores */
clean_up_IVM_hash_entry(entry, false, InvalidSubTransactionId);
@@ -1652,8 +1707,8 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
/* Set all tables in the query to pre-update state */
rewritten =3D rewrite_query_for_preupdate_state(rewritten, entry->tables,
pstate, matviewOid);
- /* Rewrite for counting duplicated tuples */
- rewritten =3D rewrite_query_for_counting(rewritten, pstate);
+ /* Rewrite for counting duplicated tuples and aggregates functions*/
+ rewritten =3D rewrite_query_for_counting_and_aggregates(rewritten, pstate=
);
=20
/* Create tuplestores to store view deltas */
if (entry->has_old)
@@ -1704,7 +1759,7 @@ IVM_immediate_maintenance(PG_FUNCTION_ARGS)
=20
count_colname =3D pstrdup("__ivm_count__");
=20
- if (query->distinctClause)
+ if (query->hasAggs || query->distinctClause)
use_count =3D true;
=20
/* calculate delta tables */
@@ -2165,17 +2220,34 @@ makeDeltaTable(RangeTblEntry *rte, MV_TriggerTable =
*table,
}
=20
/*
- * rewrite_query_for_counting
+ * rewrite_query_for_counting_and_aggregates
*
- * Rewrite query for counting duplicated tuples.
+ * Rewrite query for counting duplicated tuples and aggregate functions.
*/
static Query *
-rewrite_query_for_counting(Query *query, ParseState *pstate)
+rewrite_query_for_counting_and_aggregates(Query *query, ParseState *pstate=
)
{
TargetEntry *tle_count;
FuncCall *fn;
Node *node;
=20
+ /* For aggregate views */
+ if (query->hasAggs)
+ {
+ ListCell *lc;
+ List *aggs =3D NIL;
+ AttrNumber next_resno =3D list_length(query->targetList) + 1;
+
+ foreach(lc, query->targetList)
+ {
+ TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
+
+ if (IsA(tle->expr, Aggref))
+ makeIvmAggColumn(pstate, (Aggref *)tle->expr, tle->resname, &next_resn=
o, &aggs);
+ }
+ query->targetList =3D list_concat(query->targetList, aggs);
+ }
+
/* Add count(*) for counting distinct tuples in views */
fn =3D makeFuncCall(SystemFuncName("count"), NIL, COERCE_EXPLICIT_CALL, -=
1);
fn->agg_star =3D true;
@@ -2248,6 +2320,8 @@ rewrite_query_for_postupdate_state(Query *query, MV_T=
riggerTable *table, int rte
return query;
}
=20
+#define IVM_colname(type, col) makeObjectName("__ivm_" type, col, "_")
+
/*
* apply_delta
*
@@ -2261,6 +2335,9 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
{
StringInfoData querybuf;
StringInfoData target_list_buf;
+ StringInfo aggs_list_buf =3D NULL;
+ StringInfo aggs_set_old =3D NULL;
+ StringInfo aggs_set_new =3D NULL;
Relation matviewRel;
char *matviewname;
ListCell *lc;
@@ -2283,6 +2360,15 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tup=
lestores, Tuplestorestate *n
initStringInfo(&querybuf);
initStringInfo(&target_list_buf);
=20
+ if (query->hasAggs)
+ {
+ if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0)
+ aggs_set_old =3D makeStringInfo();
+ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0)
+ aggs_set_new =3D makeStringInfo();
+ aggs_list_buf =3D makeStringInfo();
+ }
+
/* build string of target list */
for (i =3D 0; i < matviewRel->rd_att->natts; i++)
{
@@ -2299,13 +2385,61 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
{
TargetEntry *tle =3D (TargetEntry *) lfirst(lc);
Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, i);
+ char *resname =3D NameStr(attr->attname);
=20
i++;
=20
if (tle->resjunk)
continue;
=20
- keys =3D lappend(keys, attr);
+ /*
+ * For views without aggregates, all attributes are used as keys to iden=
tify a
+ * tuple in a view.
+ */
+ if (!query->hasAggs)
+ keys =3D lappend(keys, attr);
+
+ /* For views with aggregates, we need to build SET clause for updating a=
ggregate
+ * values. */
+ if (query->hasAggs && IsA(tle->expr, Aggref))
+ {
+ Aggref *aggref =3D (Aggref *) tle->expr;
+ const char *aggname =3D get_func_name(aggref->aggfnoid);
+
+ /*
+ * We can use function names here because it is already checked if thes=
e
+ * can be used in IMMV by its OID at the definition time.
+ */
+
+ /* count */
+ if (!strcmp(aggname, "count"))
+ append_set_clause_for_count(resname, aggs_set_old, aggs_set_new, aggs_=
list_buf);
+
+ /* sum */
+ else if (!strcmp(aggname, "sum"))
+ append_set_clause_for_sum(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf);
+
+ /* avg */
+ else if (!strcmp(aggname, "avg"))
+ append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_li=
st_buf,
+ format_type_be(aggref->aggtype));
+
+ else
+ elog(ERROR, "unsupported aggregate function: %s", aggname);
+ }
+ }
+
+ /* If we have GROUP BY clause, we use its entries as keys. */
+ if (query->hasAggs && query->groupClause)
+ {
+ foreach (lc, query->groupClause)
+ {
+ SortGroupClause *sgcl =3D (SortGroupClause *) lfirst(lc);
+ TargetEntry *tle =3D get_sortgroupclause_tle(sgcl, query->targetList);
+ Form_pg_attribute attr =3D TupleDescAttr(matviewRel->rd_att, tle->resno=
- 1);
+
+ keys =3D lappend(keys, attr);
+ }
}
=20
/* Start maintaining the materialized view. */
@@ -2336,7 +2470,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
if (use_count)
/* apply old delta and get rows to be recalculated */
apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME,
- keys, count_colname);
+ keys, aggs_list_buf, aggs_set_old,
+ count_colname);
else
apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys);
=20
@@ -2362,7 +2497,7 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tupl=
estores, Tuplestorestate *n
/* apply new delta */
if (use_count)
apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME,
- keys, &target_list_buf, count_colname);
+ keys, aggs_set_new, &target_list_buf, count_colname);
else
apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf);
}
@@ -2377,6 +2512,250 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
elog(ERROR, "SPI_finish failed");
}
=20
+/*
+ * append_set_clause_for_count
+ *
+ * Append SET clause string for count aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_count(const char *resname, StringInfo buf_old,
+ StringInfo buf_new,StringInfo aggs_list)
+{
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* resname =3D mv.resname - t.resname */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", NULL, NULL));
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* resname =3D mv.resname + diff.resname */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", NULL, NULL));
+ }
+
+ appendStringInfo(aggs_list, ", %s",
+ quote_qualified_identifier("diff", resname)
+ );
+}
+
+/*
+ * append_set_clause_for_sum
+ *
+ * Append SET clause string for sum aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_sum(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list)
+{
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, resname, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, resname, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * append_set_clause_for_avg
+ *
+ * Append SET clause string for avg aggregation to given buffers.
+ * Also, append resnames required for calculating the aggregate value.
+ */
+static void
+append_set_clause_for_avg(const char *resname, StringInfo buf_old,
+ StringInfo buf_new, StringInfo aggs_list,
+ const char *aggtype)
+{
+ char *sum_col =3D IVM_colname("sum", resname);
+ char *count_col =3D IVM_colname("count", resname);
+
+ /* For tuple deletion */
+ if (buf_old)
+ {
+ /* avg =3D (mv.sum - t.sum)::aggtype / (mv.count - t.count) */
+ appendStringInfo(buf_old,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, aggtype),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+ /* sum =3D mv.sum - t.sum */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_SUB, sum_col, "mv", "t", count_col, NULL)
+ );
+ /* count =3D mv.count - t.count */
+ appendStringInfo(buf_old,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL)
+ );
+
+ }
+ /* For tuple insertion */
+ if (buf_new)
+ {
+ /* avg =3D (mv.sum + diff.sum)::aggtype / (mv.count + diff.count) */
+ appendStringInfo(buf_new,
+ ", %s =3D %s OPERATOR(pg_catalog./) %s",
+ quote_qualified_identifier(NULL, resname),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, aggtype=
),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ /* sum =3D mv.sum + diff.sum */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, sum_col),
+ get_operation_string(IVM_ADD, sum_col, "mv", "diff", count_col, NULL)
+ );
+ /* count =3D mv.count + diff.count */
+ appendStringInfo(buf_new,
+ ", %s =3D %s",
+ quote_qualified_identifier(NULL, count_col),
+ get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL)
+ );
+ }
+
+ appendStringInfo(aggs_list, ", %s, %s, %s",
+ quote_qualified_identifier("diff", resname),
+ quote_qualified_identifier("diff", IVM_colname("sum", resname)),
+ quote_qualified_identifier("diff", IVM_colname("count", resname))
+ );
+}
+
+/*
+ * get_operation_string
+ *
+ * Build a string to calculate the new aggregate values.
+ */
+static char *
+get_operation_string(IvmOp op, const char *col, const char *arg1, const ch=
ar *arg2,
+ const char* count_col, const char *castType)
+{
+ StringInfoData buf;
+ StringInfoData castString;
+ char *col1 =3D quote_qualified_identifier(arg1, col);
+ char *col2 =3D quote_qualified_identifier(arg2, col);
+ char op_char =3D (op =3D=3D IVM_SUB ? '-' : '+');
+
+ initStringInfo(&buf);
+ initStringInfo(&castString);
+
+ if (castType)
+ appendStringInfo(&castString, "::%s", castType);
+
+ if (!count_col)
+ {
+ /*
+ * If the attributes don't have count columns then calc the result
+ * by using the operator simply.
+ */
+ appendStringInfo(&buf, "(%s OPERATOR(pg_catalog.%c) %s)%s",
+ col1, op_char, col2, castString.data);
+ }
+ else
+ {
+ /*
+ * If the attributes have count columns then consider the condition
+ * where the result becomes NULL.
+ */
+ char *null_cond =3D get_null_condition_string(op, arg1, arg2, count_col)=
;
+
+ appendStringInfo(&buf,
+ "(CASE WHEN %s THEN NULL "
+ "WHEN %s IS NULL THEN %s "
+ "WHEN %s IS NULL THEN %s "
+ "ELSE (%s OPERATOR(pg_catalog.%c) %s)%s END)",
+ null_cond,
+ col1, col2,
+ col2, col1,
+ col1, op_char, col2, castString.data
+ );
+ }
+
+ return buf.data;
+}
+
+/*
+ * get_null_condition_string
+ *
+ * Build a predicate string for CASE clause to check if an aggregate value
+ * will became NULL after the given operation is applied.
+ */
+static char *
+get_null_condition_string(IvmOp op, const char *arg1, const char *arg2,
+ const char* count_col)
+{
+ StringInfoData null_cond;
+ initStringInfo(&null_cond);
+
+ switch (op)
+ {
+ case IVM_ADD:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) 0 AND %s OPERATOR(pg_catalog.=3D) 0",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ case IVM_SUB:
+ appendStringInfo(&null_cond,
+ "%s OPERATOR(pg_catalog.=3D) %s",
+ quote_qualified_identifier(arg1, count_col),
+ quote_qualified_identifier(arg2, count_col)
+ );
+ break;
+ default:
+ elog(ERROR,"unknown operation");
+ }
+
+ return null_cond.data;
+}
+
+
/*
* apply_old_delta_with_count
*
@@ -2384,13 +2763,20 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tu=
plestores, Tuplestorestate *n
* which contains tuples to be deleted from to a materialized view given b=
y
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing resnames of aggregates and SET clause for
+ * updating aggregate values.
*/
static void
apply_old_delta_with_count(const char *matviewname, const char *deltaname_=
old,
- List *keys, const char *count_colname)
+ List *keys, StringInfo aggs_list, StringInfo aggs_set,
+ const char *count_colname)
{
StringInfoData querybuf;
char *match_cond;
+ bool agg_without_groupby =3D (list_length(keys) =3D=3D 0);
=20
/* build WHERE condition for searching tuples to be deleted */
match_cond =3D get_matching_condition_string(keys);
@@ -2400,22 +2786,26 @@ apply_old_delta_with_count(const char *matviewname,=
const char *deltaname_old,
appendStringInfo(&querybuf,
"WITH t AS (" /* collecting tid of target tuples in the view */
"SELECT diff.%s, " /* count column */
- "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s) AS for_dlt, "
+ "(diff.%s OPERATOR(pg_catalog.=3D) mv.%s AND %s) AS for_dlt, "
"mv.ctid "
+ "%s " /* aggregate columns */
"FROM %s AS mv, %s AS diff "
"WHERE %s" /* tuple matching condition */
"), updt AS (" /* update a tuple if this is not to be deleted */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.-) t.%s "
+ "%s" /* SET clauses for aggregates */
"FROM t WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND NOT for_dl=
t "
")"
/* delete a tuple if this is to be deleted */
"DELETE FROM %s AS mv USING t "
"WHERE mv.ctid OPERATOR(pg_catalog.=3D) t.ctid AND for_dlt",
count_colname,
- count_colname, count_colname,
+ count_colname, count_colname, (agg_without_groupby ? "false" : "true"=
),
+ (aggs_list !=3D NULL ? aggs_list->data : ""),
matviewname, deltaname_old,
match_cond,
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
matviewname);
=20
if (SPI_exec(querybuf.data, 0) !=3D SPI_OK_DELETE)
@@ -2479,10 +2869,15 @@ apply_old_delta(const char *matviewname, const char=
*deltaname_old,
* matviewname. This is used when counting is required, that is, the view
* has aggregate or distinct. Also, when a table in EXISTS sub queries
* is modified.
+ *
+ * If the view desn't have aggregates or has GROUP BY, this requires a key=
s
+ * list to identify a tuple in the view. If the view has aggregates, this
+ * requires strings representing SET clause for updating aggregate values.
*/
static void
apply_new_delta_with_count(const char *matviewname, const char* deltaname_=
new,
- List *keys, StringInfo target_list, const char* count_colname)
+ List *keys, StringInfo aggs_set, StringInfo target_list,
+ const char* count_colname)
{
StringInfoData querybuf;
StringInfoData returning_keys;
@@ -2513,6 +2908,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
appendStringInfo(&querybuf,
"WITH updt AS (" /* update a tuple if this exists in the view */
"UPDATE %s AS mv SET %s =3D mv.%s OPERATOR(pg_catalog.+) diff.%s "
+ "%s " /* SET clauses for aggregates */
"FROM %s AS diff "
"WHERE %s " /* tuple matching condition */
"RETURNING %s" /* returning keys of updated tuples */
@@ -2520,6 +2916,7 @@ apply_new_delta_with_count(const char *matviewname, c=
onst char* deltaname_new,
"SELECT %s FROM %s AS diff "
"WHERE NOT EXISTS (SELECT 1 FROM updt AS mv WHERE %s);",
matviewname, count_colname, count_colname, count_colname,
+ (aggs_set !=3D NULL ? aggs_set->data : ""),
deltaname_new,
match_cond,
returning_keys.data,
diff --git a/src/include/commands/createas.h b/src/include/commands/createa=
s.h
index bfd0249b10d..313b129f7e8 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -30,6 +30,7 @@ extern void CreateIvmTriggersOnBaseTables(Query *qry, Oid=
matviewOid);
extern void CreateIndexOnIMMV(Query *query, Relation matviewRel);
=20
extern Query *rewriteQueryForIMMV(Query *query, List *colNames);
+extern void makeIvmAggColumn(ParseState *pstate, Aggref *aggref, char *res=
name, AttrNumber *next_resno, List **aggs);
=20
extern int GetIntoRelEFlags(IntoClause *intoClause);
=20
--=20
2.43.0
--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ
Content-Type: text/x-diff;
name="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Disposition: attachment;
filename="v38-0007-Add-DISTINCT-support-for-IVM.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 33+ messages in thread
* RE: Popcount optimization using AVX512
@ 2024-01-25 05:43 Shankaran, Akash <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Shankaran, Akash @ 2024-01-25 05:43 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Noah Misch <[email protected]>; Amonson, Paul D <[email protected]>; Tom Lane <[email protected]>; Matthias van de Meent <[email protected]>; [email protected] <[email protected]>
Sorry for the late response. We did some further testing and research on our end, and ended up modifying the AVX512 based algorithm for popcount. We removed a scalar dependency and accumulate the results of popcnt instruction in a zmm register, only performing the reduce add at the very end, similar to [0].
With the updated patch, we observed significant improvements and handily beat the previous popcount algorithm performance. No regressions in any scenario are observed:
Platform: Intel Xeon Platinum 8360Y (Icelake) for data sizes 1kb - 64kb.
Microbenchmark: 2x - 3x gains presently vs 19% previously, on the same microbenchmark described initially in this thread.
PG testing:
SQL bit_count() calls popcount. Using a Postgres benchmark calling "select bit_count(bytea(col1)) from mytable" on a table with ~2M text rows, each row 1-12kb in size, we observe (only comparing with 64bit PG implementation, which is the fastest):
1. Entire benchmark using AVX512 implementation vs PG 64-bit impl runs 6-13% faster.
2. Reduce time spent on pg_popcount() method in postgres server during the benchmark:
o 64bit (current PG): 29.5%
o AVX512: 3.3%
3. Reduce number of samples processed by popcount:
o 64bit (current PG): 2.4B samples
o AVX512: 285M samples
Compile above patch (on a machine supporting AVX512 vpopcntdq) using: make all CFLAGS_AVX512="-DHAVE__HW_AVX512_POPCNT -mavx -mavx512vpopcntdq -mavx512f -march=native
Attaching flamegraphs and patch for above observations.
[0] https://github.com/WojciechMula/sse-popcount/blob/master/popcnt-avx512-vpopcnt.cpp
Thanks,
Akash Shankaran
-----Original Message-----
From: Nathan Bossart <[email protected]>
Sent: Wednesday, November 15, 2023 1:49 PM
To: Shankaran, Akash <[email protected]>
Cc: Noah Misch <[email protected]>; Amonson, Paul D <[email protected]>; Tom Lane <[email protected]>; Matthias van de Meent <[email protected]>; [email protected]
Subject: Re: Popcount optimization using AVX512
On Wed, Nov 15, 2023 at 08:27:57PM +0000, Shankaran, Akash wrote:
> AVX512 has light and heavy instructions. While the heavy AVX512
> instructions have clock frequency implications, the light instructions
> not so much. See [0] for more details. We captured EMON data for the
> benchmark used in this work, and see that the instructions are using
> the licensing level not meant for heavy AVX512 operations. This means
> the instructions for popcount : _mm512_popcnt_epi64(),
> _mm512_reduce_add_epi64() are not going to have any significant impact
> on CPU clock frequency.
>
> Clock frequency impact aside, we measured the same benchmark for gains
> on older Intel hardware and observe up to 18% better performance on
> Intel Icelake. On older intel hardware, the popcntdq 512 instruction
> is not present so it won’t work. If clock frequency is not affected,
> rest of workload should not be impacted in the case of mixed workloads.
Thanks for sharing your analysis.
> Testing this on smaller block sizes < 8KiB shows that AVX512 compared
> to the current 64bit behavior shows slightly lower performance, but
> with a large variance. We cannot conclude much from it. The testing
> with ANALYZE benchmark by Nathan also points to no visible impact as a
> result of using AVX512. The gains on larger dataset is easily evident,
> with less variance.
>
> What are your thoughts if we introduce AVX512 popcount for smaller
> sizes as an optional feature initially, and then test it more
> thoroughly over time on this particular use case?
I don't see any need to rush this. At the very earliest, this feature would go into v17, which doesn't enter feature freeze until April 2024.
That seems like enough time to complete any additional testing you'd like to do. However, if you are seeing worse performance with this patch, then it seems unlikely that we'd want to proceed.
> Thoughts or feedback on the approach in the patch? This solution
> should not impact anyone who doesn’t use the feature i.e. AVX512. Open
> to additional ideas if this doesn’t seem like the right approach here.
It's true that it wouldn't impact anyone not using the feature, but there's also a decent chance that this code goes virtually untested. As I've stated elsewhere [0], I think we should ensure there's buildfarm coverage for this kind of architecture-specific stuff.
[0] https://postgr.es/m/20230726043707.GB3211130%40nathanxps13
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] perf-avx512-1.8mrows.svg (154.0K, ../../PH0PR11MB5000C2258BF2804AAF7AAE27F27A2@PH0PR11MB5000.namprd11.prod.outlook.com/2-perf-avx512-1.8mrows.svg)
download
[application/octet-stream] perf-with-64bit-1.8m.svg (138.2K, ../../PH0PR11MB5000C2258BF2804AAF7AAE27F27A2@PH0PR11MB5000.namprd11.prod.outlook.com/3-perf-with-64bit-1.8m.svg)
download
[application/octet-stream] popcount_avx512.patch (1.9K, ../../PH0PR11MB5000C2258BF2804AAF7AAE27F27A2@PH0PR11MB5000.namprd11.prod.outlook.com/4-popcount_avx512.patch)
download | inline diff:
diff --git a/src/port/Makefile b/src/port/Makefile
index dcc8737e68..354ab636da 100644
--- a/src/port/Makefile
+++ b/src/port/Makefile
@@ -87,6 +87,11 @@ pg_crc32c_sse42.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_sse42_shlib.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_sse42_srv.o: CFLAGS+=$(CFLAGS_CRC)
+# Newer Intel processors can use some AVX-512 Capabilities (11/01/2023)
+pg_bitutils.o: CFLAGS+=$(CFLAGS_AVX512)
+pg_bitutils_shlib.o: CFLAGS+=$(CFLAGS_AVX512)
+pg_bitutils_srv.o:CFLAGS+=$(CFLAGS_AVX512)
+
# all versions of pg_crc32c_armv8.o need CFLAGS_CRC
pg_crc32c_armv8.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_armv8_shlib.o: CFLAGS+=$(CFLAGS_CRC)
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index 640a89561a..a0c91273ec 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -19,6 +19,10 @@
#include <intrin.h>
#endif
+#if defined(HAVE__HW_AVX512_POPCNT)
+#include <immintrin.h>
+#endif
+
#include "port/pg_bitutils.h"
@@ -298,6 +302,23 @@ pg_popcount(const char *buf, int bytes)
uint64 popcnt = 0;
#if SIZEOF_VOID_P >= 8
+#if defined(HAVE__HW_AVX512_POPCNT)
+ uint64 tmp[8] __attribute__((aligned(64)));
+ __m512i *pc_result = (__m512i *)tmp;
+ __m512i accumulator = _mm512_setzero_si512();
+ while (bytes >= 64)
+ {
+ const __m512i v = _mm512_loadu_si512((const __m512i *)buf);
+ const __m512i p = _mm512_popcnt_epi64(v);
+ accumulator = _mm512_add_epi64(accumulator, p);
+ bytes -= 64;
+ buf += 64;
+ }
+ _mm512_store_si512(pc_result, accumulator);
+ popcnt = _mm512_reduce_add_epi64(*pc_result);
+ bytes = bytes % 64;
+
+#else // HAVE__HW_AVX512_POPCNT
/* Process in 64-bit chunks if the buffer is aligned. */
if (buf == (const char *) TYPEALIGN(8, buf))
{
@@ -311,6 +332,7 @@ pg_popcount(const char *buf, int bytes)
buf = (const char *) words;
}
+#endif // HAVE__HW_AVX512_POPCNT
#else
/* Process in 32-bit chunks if the buffer is aligned. */
if (buf == (const char *) TYPEALIGN(4, buf))
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Popcount optimization using AVX512
@ 2024-01-25 09:49 Alvaro Herrera <[email protected]>
parent: Shankaran, Akash <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Alvaro Herrera @ 2024-01-25 09:49 UTC (permalink / raw)
To: Shankaran, Akash <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Noah Misch <[email protected]>; Amonson, Paul D <[email protected]>; Tom Lane <[email protected]>; Matthias van de Meent <[email protected]>; [email protected] <[email protected]>
On 2024-Jan-25, Shankaran, Akash wrote:
> With the updated patch, we observed significant improvements and
> handily beat the previous popcount algorithm performance. No
> regressions in any scenario are observed:
> Platform: Intel Xeon Platinum 8360Y (Icelake) for data sizes 1kb - 64kb.
> Microbenchmark: 2x - 3x gains presently vs 19% previously, on the same
> microbenchmark described initially in this thread.
These are great results.
However, it would be much better if the improved code were available for
all relevant builds and activated if a CPUID test determines that the
relevant instructions are available, instead of requiring a compile-time
flag -- which most builds are not going to use, thus wasting the
opportunity for running the optimized code.
I suppose this would require patching pg_popcount64_choose() to be more
specific. Looking at the existing code, I would also consider renaming
the "_fast" variants to something like pg_popcount32_asml/
pg_popcount64_asmq so that you can name the new one pg_popcount64_asmdq
or such. (Or maybe leave the 32-bit version alone as "fast/slow", since
there's no third option for that one -- or do I misread?)
I also think this needs to move the CFLAGS-decision-making elsewhere;
asking the user to get it right is too much of a burden. Is it workable
to simply verify compiler support for the additional flags needed, and
if so add them to a new CFLAGS_BITUTILS variable or such? We already
have the CFLAGS_CRC model that should be easy to follow. Should be easy
enough to mostly copy what's in configure.ac and meson.build, right?
Finally, the matter of using ifunc as proposed by Noah seems to be still
in the air, with no patches offered for the popcount family. Given that
Nathan reports [1] a performance decrease, maybe we should set that
thought aside for now and continue to use function pointers. It's worth
keeping in mind that popcount is already using function pointers (at
least in the case where we try to use POPCNT directly), so patching to
select between three options instead of between two wouldn't be a
regression.
[1] https://postgr.es/m/20231107201441.GA898662@nathanxps13
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"Nunca se desea ardientemente lo que solo se desea por razón" (F. Alexandre)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Popcount optimization using AVX512
@ 2024-01-26 06:42 Alvaro Herrera <[email protected]>
parent: Alvaro Herrera <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Alvaro Herrera @ 2024-01-26 06:42 UTC (permalink / raw)
To: Shankaran, Akash <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Noah Misch <[email protected]>; Amonson, Paul D <[email protected]>; Tom Lane <[email protected]>; Matthias van de Meent <[email protected]>; [email protected] <[email protected]>
On 2024-Jan-25, Alvaro Herrera wrote:
> Finally, the matter of using ifunc as proposed by Noah seems to be still
> in the air, with no patches offered for the popcount family.
Oh, I just realized that the patch as currently proposed is placing the
optimized popcount code in the path that does not require going through
a function pointer. So the performance increase is probably coming from
both avoiding jumping through the pointer as well as from the improved
instruction.
This suggests that finding a way to make the ifunc stuff work (with good
performance) is critical to this work.
--
Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/
"The ability of users to misuse tools is, of course, legendary" (David Steele)
https://postgr.es/m/[email protected]
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Why does exec_simple_query requires 2 snapshots
@ 2025-02-19 01:10 Andy Fan <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Andy Fan @ 2025-02-19 01:10 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: [email protected] <[email protected]>
Tom Lane <[email protected]> writes:
> Andy Fan <[email protected]> writes:
>> My question is why can't we share the same snapshot for the 2 cases?
>> parser & planner requires Catalog Snapshot which should be the
>> latest one, but in the above case, looks the executor can reuse it as
>> well. Is there anything I missed?
>
> Many years ago, we committed a patch to do exactly that. We had
> to back it out again because it broke too many real-world scenarios.
> I'm too lazy to search the archives for you, but you might be able
> to find the commit and revert by searching the git history.
>
> regards, tom lane
Commit id is 532994299e2, thank you Tom!
--
Best Regards
Andy Fan
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Why does exec_simple_query requires 2 snapshots
@ 2025-02-19 02:30 Tom Lane <[email protected]>
parent: Andy Fan <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Tom Lane @ 2025-02-19 02:30 UTC (permalink / raw)
To: Andy Fan <[email protected]>; +Cc: [email protected] <[email protected]>
Andy Fan <[email protected]> writes:
> Tom Lane <[email protected]> writes:
>> Many years ago, we committed a patch to do exactly that. We had
>> to back it out again because it broke too many real-world scenarios.
>> I'm too lazy to search the archives for you, but you might be able
>> to find the commit and revert by searching the git history.
> Commit id is 532994299e2, thank you Tom!
Ah, cool. If you didn't find it already, there should be a
pgsql-hackers or pgsql-bugs thread of approximately that date
with more detail. Sadly, we didn't adopt the convention of
including Discussion: links in commit messages till some
years after that.
regards, tom lane
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Why does exec_simple_query requires 2 snapshots
@ 2025-02-19 08:50 Andy Fan <[email protected]>
parent: Tom Lane <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Andy Fan @ 2025-02-19 08:50 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: [email protected] <[email protected]>
Tom Lane <[email protected]> writes:
> Andy Fan <[email protected]> writes:
>> Tom Lane <[email protected]> writes:
>>> Many years ago, we committed a patch to do exactly that. We had
>>> to back it out again because it broke too many real-world scenarios.
>>> I'm too lazy to search the archives for you, but you might be able
>>> to find the commit and revert by searching the git history.
>
>> Commit id is 532994299e2, thank you Tom!
>
> links in commit messages till some years after that.
Links in commit messages is a really great change.
I have found more details at [1], the issue can be summaried as IIUC:
The current strategy is below:
t1: snapshot1
parser&planner
t2: snapshot2
executor.
which makes the committed tuple between (t1, t2] are visible to user, but if
we share the same snapshot, the committed tuple would not be visible to
user, Chaning it would be a behavior change.
[1] https://www.postgresql.org/message-id/5075D8DF.6050500%40fuzzy.cz
--
Best Regards
Andy Fan
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Why does exec_simple_query requires 2 snapshots
@ 2025-03-08 09:51 Michail Nikolaev <[email protected]>
parent: Andy Fan <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Michail Nikolaev @ 2025-03-08 09:51 UTC (permalink / raw)
To: Andy Fan <[email protected]>; +Cc: Tom Lane <[email protected]>; [email protected] <[email protected]>
Hello!
I was thinking the same thing once - and you may
see db989184cda7f4aa1ff764cca96142029e7e093b for the special comment about
that :)
https://github.com/postgres/postgres/commit/db989184cda7f4aa1ff764cca96142029e7e093b
^ permalink raw reply [nested|flat] 33+ messages in thread
end of thread, other threads:[~2025-03-08 09:51 UTC | newest]
Thread overview: 33+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-08-01 19:59 [PATCH 1/3] Avoid GIN full scan for empty ALL keys Nikita Glukhov <[email protected]>
2019-11-15 14:15 [PATCH 1/5] Avoid GIN full scan for empty ALL keys Nikita Glukhov <[email protected]>
2021-08-02 05:59 [PATCH v27 7/9] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v25 08/15] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v27 7/9] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v23 08/15] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v24 08/15] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v24 08/15] Add aggregates support in IVM Yugo Nagata <[email protected]>
2021-08-02 05:59 [PATCH v26 08/10] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v29 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v31 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v37 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v37 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v38 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v28 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v38 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v38 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v29 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v30 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v37 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v29 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v29 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v32 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v30 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v30 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2023-05-31 11:46 [PATCH v30 08/11] Add aggregates support in IVM Yugo Nagata <[email protected]>
2024-01-25 05:43 RE: Popcount optimization using AVX512 Shankaran, Akash <[email protected]>
2024-01-25 09:49 ` Re: Popcount optimization using AVX512 Alvaro Herrera <[email protected]>
2024-01-26 06:42 ` Re: Popcount optimization using AVX512 Alvaro Herrera <[email protected]>
2025-02-19 01:10 Re: Why does exec_simple_query requires 2 snapshots Andy Fan <[email protected]>
2025-02-19 02:30 ` Re: Why does exec_simple_query requires 2 snapshots Tom Lane <[email protected]>
2025-02-19 08:50 ` Re: Why does exec_simple_query requires 2 snapshots Andy Fan <[email protected]>
2025-03-08 09:51 ` Re: Why does exec_simple_query requires 2 snapshots Michail Nikolaev <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox